diff --git a/lib/commands/stop.mjs b/lib/commands/stop.mjs
index 08bb6d41..655e91e3 100644
--- a/lib/commands/stop.mjs
+++ b/lib/commands/stop.mjs
@@ -77,8 +77,8 @@ export async function sendStopRunRequest(runId, apiToken) {
try {
log.debug(`Sending stop Run with id: ${runId} request`);
- await WebConsoleClient.sendPathRequest({
- path: `load/runs/${runId}`,
+ await WebConsoleClient.sendGetRequest({
+ path: `load/runs/${runId}/stop`,
qParams: {
api_token: apiToken
}
diff --git a/lib/config/env.properties b/lib/config/env.properties
index 56637125..c046dde5 100644
--- a/lib/config/env.properties
+++ b/lib/config/env.properties
@@ -7,11 +7,6 @@ BROWSERUP_LOCKBOX_MASTER_KEY=439e901f30a3ce31836dbc81e67411e1f84f43491740ca0f91d
BROWSERUP_GRID_JAVA_API_HOST=grid-java-api
BROWSERUP_GRID_JAVA_API_PORT=23001
-BROWSERUP_MYSQL_HOST=mysql
-BROWSERUP_MYSQL_PORT=23002
-BROWSERUP_MYSQL_PASSWORD=just_testing
-BROWSERUP_MYSQL_USERNAME=browserup
-
BROWSERUP_S3_MINIO_HOST=minio
BROWSERUP_S3_MINIO_PORT_1=23003
BROWSERUP_S3_MINIO_PORT_2=23004
@@ -28,19 +23,12 @@ BROWSERUP_ZOOKEEPER_HOST=zookeeper
BROWSERUP_ZOOKEEPER_CLIENT_PORT=23008
BROWSERUP_ZOO_MAX_CLIENT_CNXNS=500
-BROWSERUP_REDIS_HOST=redis
-BROWSERUP_REDIS_PORT=23009
+BROWSERUP_CLICKHOUSE_HOST=clickhouse
BROWSERUP_CHRONOGRAF_HOST=chronograf
BROWSERUP_CHRONOGRAF_PORT=23010
-BROWSERUP_INFLUX_DB_HOST=influxdb
-BROWSERUP_INFLUX_DB_PORT=23011
-BROWSERUP_INFLUX_DB_NAME=browserup
-BROWSERUP_INFLUX_DB_USER=browserup
-BROWSERUP_INFLUX_TEST_DB_HOST=influx-test-db
-BROWSERUP_INFLUX_TEST_DB_PORT=23011
-BROWSERUP_INFLUX_DB_PASSWORD=just_testing
+BROWSERUP_LOKI_HOST=loki
BROWSERUP_S3_MINIO_HOST_PORT_1=23012
@@ -50,3 +38,6 @@ BROWSERUP_GRAFANA_PASSWORD=just_testing
BROWSERUP_PRIVATE_DNS_NAMESPACE_NAME=browserup-internal.com
BROWSERUP_CLUSTER_PUBLIC_IP=localhost
+
+BROWSERUP_NATS_URLS=nats://nats:4222
+BROWSERUP_NATS_SERVER_LIST=nats:4222
diff --git a/lib/docker_compose/docker-compose.yml b/lib/docker_compose/docker-compose.yml
index 50c220f2..77422818 100644
--- a/lib/docker_compose/docker-compose.yml
+++ b/lib/docker_compose/docker-compose.yml
@@ -6,11 +6,7 @@ services:
backend-services:
image: busybox
depends_on:
- - chronograf
- - influxdb
- - mysql
- rabbitmq
- - redis
- zookeeper
- minio
- grafana
@@ -18,8 +14,6 @@ services:
backend-services-nodb:
image: busybox
depends_on:
- - chronograf
- - influxdb
- rabbitmq
- zookeeper
@@ -61,12 +55,7 @@ services:
- BROWSERUP_RABBITMQ_DEFAULT_PASS=${BROWSERUP_RABBITMQ_DEFAULT_PASS}
- BROWSERUP_RABBITMQ_QUEUE_PORT=5672
- BROWSERUP_ZOOKEEPER_HOST=${BROWSERUP_ZOOKEEPER_HOST}
- - BROWSERUP_ZOOKEEPER_CLIENT_PORT=2181
- - BROWSERUP_INFLUX_DB_HOST=${BROWSERUP_INFLUX_DB_HOST}
- - BROWSERUP_INFLUX_DB_PORT=8086
- - BROWSERUP_INFLUX_DB_NAME=browserup
- - BROWSERUP_INFLUX_DB_USER=browserup
- - BROWSERUP_INFLUX_DB_PASSWORD=${BROWSERUP_INFLUX_DB_PASSWORD}
+ - BROWSERUP_ZOOKEEPER_CLIENT_PORT=9181
- BROWSERUP_MINION_VUS=1
- BROWSERUP_MAX_MINIONS_PER_EC2_INSTANCE=${BROWSERUP_MAX_MINIONS_PER_EC2_INSTANCE}
- BROWSERUP_IS_CLOUD=false
@@ -88,8 +77,8 @@ services:
- BROWSERUP_S3_MINIO_HOST_PORT_1=${BROWSERUP_S3_MINIO_PORT_1}
- BROWSERUP_S3_MINIO_ARTIFACTS_BUCKET=${BROWSERUP_S3_MINIO_ARTIFACTS_BUCKET}
- BROWSERUP_CLUSTER_NAME=${BROWSERUP_CLUSTER_NAME}
+ - BROWSERUP_NATS_URLS=${BROWSERUP_NATS_URLS}
depends_on:
- - influxdb
- zookeeper
- rabbitmq
- fluentbit
@@ -119,12 +108,7 @@ services:
- BROWSERUP_RABBITMQ_DEFAULT_PASS=${BROWSERUP_RABBITMQ_DEFAULT_PASS}
- BROWSERUP_RABBITMQ_QUEUE_PORT=5672
- BROWSERUP_ZOOKEEPER_HOST=${BROWSERUP_ZOOKEEPER_HOST}
- - BROWSERUP_ZOOKEEPER_CLIENT_PORT=2181
- - BROWSERUP_INFLUX_DB_HOST=${BROWSERUP_INFLUX_DB_HOST}
- - BROWSERUP_INFLUX_DB_PORT=8086
- - BROWSERUP_INFLUX_DB_NAME=browserup
- - BROWSERUP_INFLUX_DB_USER=browserup
- - BROWSERUP_INFLUX_DB_PASSWORD=${BROWSERUP_INFLUX_DB_PASSWORD}
+ - BROWSERUP_ZOOKEEPER_CLIENT_PORT=9181
- BROWSERUP_MINION_VUS=1
- BROWSERUP_MINION_IMAGE_VERSION_TAG=${MINION_IMAGE_VERSION_TAG}
- BROWSERUP_S3_MINIO_ACCESS_KEY_ID=${BROWSERUP_S3_MINIO_ACCESS_KEY_ID}
@@ -138,6 +122,33 @@ services:
- BROWSERUP_CLUSTER_NAME=${BROWSERUP_CLUSTER_NAME}
- BROWSERUP_USER_ARTIFACT_CACHE_PATH=/bu/artifactcache
- BROWSERUP_USE_LOCAL_DOCKER_CACHED_IMAGE=${BROWSERUP_USE_LOCAL_DOCKER_CACHED_IMAGE}
+ - BROWSERUP_NATS_URLS=${BROWSERUP_NATS_URLS}
+ depends_on:
+ - zookeeper
+ - rabbitmq
+ - grid-java-api
+ - fluentbit
+ - clickhouse
+ logging:
+ driver: "fluentd"
+ options:
+ fluentd-address: "localhost:24224"
+
+ grid-java-observer:
+ container_name: browserup_grid-java-observer
+ image: docker.io/browserup/observer:latest
+ volumes:
+ - /var/run/docker.sock:/var/run/docker.sock
+ restart: on-failure
+ environment:
+ - BROWSERUP_RABBITMQ_HOST=${BROWSERUP_RABBITMQ_HOST}
+ - BROWSERUP_RABBITMQ_DEFAULT_USER=browserup
+ - BROWSERUP_RABBITMQ_DEFAULT_PASS=${BROWSERUP_RABBITMQ_DEFAULT_PASS}
+ - BROWSERUP_RABBITMQ_QUEUE_PORT=5672
+ - BROWSERUP_ZOOKEEPER_HOST=${BROWSERUP_ZOOKEEPER_HOST}
+ - BROWSERUP_ZOOKEEPER_CLIENT_PORT=9181
+ - BROWSERUP_IS_CLOUD=false
+ - BROWSERUP_NATS_URLS=${BROWSERUP_NATS_URLS}
depends_on:
- zookeeper
- rabbitmq
@@ -150,7 +161,7 @@ services:
webconsole:
container_name: browserup_webconsole
- image: docker.io/browserup/webconsole:latest
+ image: docker.io/browserup/webconsole:snapshot
privileged: true
security_opt:
- "seccomp=unconfined"
@@ -158,21 +169,13 @@ services:
devices:
- "/dev/fuse"
ports:
- - "${BROWSERUP_WEBCONSOLE_PORT}:3000"
+ - "${BROWSERUP_WEBCONSOLE_PORT}:3100"
environment:
- DOCKER_LOGS=true
- - BROWSERUP_MYSQL_HOST=${BROWSERUP_MYSQL_HOST}
- - BROWSERUP_MYSQL_USERNAME=browserup
- - BROWSERUP_MYSQL_PASSWORD=${BROWSERUP_MYSQL_PASSWORD}
- - BROWSERUP_MYSQL_PORT=3306
- - BROWSERUP_INFLUX_DB_HOST=${BROWSERUP_INFLUX_DB_HOST}
- - BROWSERUP_INFLUX_DB_PORT=8086
- - BROWSERUP_INFLUX_DB_NAME=browserup
- - BROWSERUP_INFLUX_DB_USER=browserup
- - BROWSERUP_INFLUX_DB_PASSWORD=${BROWSERUP_INFLUX_DB_PASSWORD}
+ - BROWSERUP_CLICKHOUSE_HOST=${BROWSERUP_CLICKHOUSE_HOST}
- BROWSERUP_GRID_SERVICE_URL=http://${BROWSERUP_GRID_JAVA_API_HOST}:8080/grid
+ - BROWSERUP_LOKI_SERVICE_URL=http://${BROWSERUP_LOKI_HOST}:3100
- BROWSERUP_IS_CLOUD=false
- - BROWSERUP_REDIS_URL=redis://${BROWSERUP_REDIS_HOST}:6379/1
- BROWSERUP_S3_MINIO_ACCESS_KEY_ID=${BROWSERUP_S3_MINIO_ACCESS_KEY_ID}
- BROWSERUP_S3_MINIO_SECRET_ACCESS_KEY=${BROWSERUP_S3_MINIO_SECRET_ACCESS_KEY}
- BROWSERUP_S3_MINIO_ARTIFACTS_BUCKET=${BROWSERUP_S3_MINIO_ARTIFACTS_BUCKET}
@@ -186,68 +189,50 @@ services:
- RAILS_LOG_TO_STDOUT=true
- RAILS_SERVE_STATIC_FILES=true
- BROWSERUP_S3_MINIO_HOST_PORT_1=${BROWSERUP_S3_MINIO_PORT_1}
+ - RAILS_MASTER_KEY=6f893b421a180634146cd1eb5f58cc80
+ - PORT=3000
+ - BROWSERUP_WC_RUNNING_AS_SERVER=true
+ - BROWSERUP_NATS_URLS=${BROWSERUP_NATS_URLS}
+ - BROWSERUP_NATS_SERVER_LIST=${BROWSERUP_NATS_SERVER_LIST}
depends_on:
- - mysql
- - influxdb
- grid-java-api
- - redis
- minio
- fluentbit
logging:
driver: "fluentd"
options:
fluentd-address: "localhost:24224"
- volumes:
- - ./containers:/var/lib/containers
- chronograf:
- image: chronograf:1.7.16
- ports:
- - "${BROWSERUP_CHRONOGRAF_PORT}:8888"
- depends_on:
- - influxdb
- - fluentbit
- logging:
- driver: "fluentd"
- options:
- fluentd-address: "localhost:24224"
-
- influxdb:
- container_name: browserup_influxdb
- image: influxdb:1.8.10
+ clickhouse:
+ image: 'clickhouse/clickhouse-server:25.1'
+ container_name: clickhouse
+ hostname: clickhouse
volumes:
- - influxdb-volume:/var/lib/influxdb
- ports:
- - "${BROWSERUP_INFLUX_DB_PORT}:8086"
- environment:
- - INFLUXDB_DB=browserup
- - INFLUX_DB_USER=browserup
- - INFLUX_DB_PASSWORD=${BROWSERUP_INFLUX_DB_PASSWORD}
- - INFLUXDB_LOGGING_LEVEL=debug
- - INFLUXDB_DATA_MAX_SERIES_PER_DATABASE=0
- - INFLUXDB_DATA_MAX_VALUES_PER_TAG=0
- - INFLUXDB_HTTP_LOG_ENABLED=false
+ - ./resources/click-house/config.xml:/etc/clickhouse-server/config.d/config.xml
+ - ./resources/click-house/users.xml:/etc/clickhouse-server/users.d/users.xml
+ - ./resources/click-house/schemas/errors.proto:/var/lib/clickhouse/format_schemas/errors.proto
+ - ./resources/click-house/schemas/run_infos.proto:/var/lib/clickhouse/format_schemas/run_infos.proto
+ - ./resources/click-house/schemas/steps.proto:/var/lib/clickhouse/format_schemas/steps.proto
+ - ./resources/click-house/schemas/system_stats.proto:/var/lib/clickhouse/format_schemas/system_stats.proto
+ - ./resources/click-house/schemas/urls.proto:/var/lib/clickhouse/format_schemas/urls.proto
+ - ./resources/click-house/schemas/verifications.proto:/var/lib/clickhouse/format_schemas/verifications.proto
+ - ./resources/click-house/schemas/vu_logs.proto:/var/lib/clickhouse/format_schemas/vu_logs.proto
+ - ./resources/click-house/schemas/vu_statuses.proto:/var/lib/clickhouse/format_schemas/vu_statuses.proto
depends_on:
+ - zookeeper
+ - nats
- fluentbit
logging:
driver: "fluentd"
options:
fluentd-address: "localhost:24224"
- mysql:
- container_name: browserup_mysql
- image: mysql:8.0.31-oracle
- command: mysqld --default-authentication-plugin=mysql_native_password
+ zookeeper:
+ image: 'clickhouse/clickhouse-keeper:24.4'
+ container_name: zookeeper
+ hostname: zookeeper
volumes:
- - mysql-volume:/var/lib/mysql
- - ./resources/init.sql:/docker-entrypoint-initdb.d/z-create-test-db.sql
- ports:
- - "${BROWSERUP_MYSQL_PORT}:3306"
- environment:
- - MYSQL_RANDOM_ROOT_PASSWORD=yes
- - MYSQL_DATABASE=browserup_development
- - MYSQL_USER=browserup
- - MYSQL_PASSWORD=${BROWSERUP_MYSQL_PASSWORD}
+ - ./resources/click-house/keeper-config.xml:/etc/clickhouse-keeper/keeper_config.xml
depends_on:
- fluentbit
logging:
@@ -255,17 +240,11 @@ services:
options:
fluentd-address: "localhost:24224"
- redis:
- container_name: browserup_redis
- image: redis:6.2.5
- ports:
- - "${BROWSERUP_REDIS_PORT}:6379"
- depends_on:
- - fluentbit
- logging:
- driver: "fluentd"
- options:
- fluentd-address: "localhost:24224"
+ nats:
+ image: nats
+ volumes:
+ - ./resources/nats/nats-server.conf:/etc/nats/nats-server.conf
+ command: "--cluster_name NATS --cluster nats://0.0.0.0:6222 --http_port 8222 -c /etc/nats/nats-server.conf"
rabbitmq:
container_name: browserup_rabbitmq
@@ -291,28 +270,6 @@ services:
options:
fluentd-address: "localhost:24224"
- zookeeper:
- container_name: browserup_zookeeper
- image: zookeeper:3.8.1
- volumes:
- - ./resources/zoo.cfg:/conf/zoo.cfg
- - zookeeper-data-volume:/data
- - zookeeper-datalog-volume:/datalog
- ports:
- - "${BROWSERUP_ZOOKEEPER_CLIENT_PORT}:2181"
- environment:
- - ZOO_TICK_TIME=300000 # Helpful for debugging
- - ZOO_CFG_EXTRA=warn
- - ZOO_MAX_CLIENT_CNXNS=${BROWSERUP_ZOO_MAX_CLIENT_CNXNS}
- depends_on:
- - fluentbit
- - loki
- - grafana
- logging:
- driver: "fluentd"
- options:
- fluentd-address: "localhost:24224"
-
minio:
container_name: browserup_minio
image: quay.io/minio/minio:RELEASE.2022-01-08T03-11-54Z
@@ -336,17 +293,13 @@ services:
# router
fluentbit:
- image: fluent/fluent-bit:2.1.10
+ image: fluent/fluent-bit:3.0.3
depends_on:
- loki
environment:
- LOG_PATH=/tmp
- ports:
- - "24224:24224"
- - "24224:24224/udp"
- - "2020:2020"
volumes:
- - ./resources/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
+ - ./resources/fluent-bit:/fluent-bit/etc
# storage
loki:
@@ -371,6 +324,18 @@ services:
- ./resources/grafana-default-dashboard.json:/etc/grafana/provisioning/dashboards/grafana-default-dashboard.json
- ./resources/grafana-zookeeper-dashboard.json:/etc/grafana/provisioning/dashboards/grafana-zookeeper-dashboard.json
- ./resources/grafana-grid-services-dashboard.json:/etc/grafana/provisioning/dashboards/grafana-grid-services-dashboard.json
+ - ./resources/grafana-cadvisor-dashboard.json:/etc/grafana/provisioning/dashboards/grafana-cadvisor-dashboard.json
+ - ./resources/grafana-clickhouse-dashboard.json:/etc/grafana/provisioning/dashboards/grafana-clickhouse-dashboard.json
+
+ cadvisor:
+ image: gcr.io/cadvisor/cadvisor
+ container_name: cadvisor
+ volumes:
+ - /:/rootfs:ro
+ - /var/run:/var/run:ro
+ - /sys:/sys:ro
+ - /var/lib/docker/:/var/lib/docker:ro
+ - /dev/disk:/dev/disk/:ro
prometheus:
image: prom/prometheus:v2.44.0
@@ -380,8 +345,6 @@ services:
- "${PROMETHEUS_PORT}:9090"
volumes:
- influxdb-volume:
- mysql-volume:
rabbitmq-volume:
zookeeper-data-volume:
zookeeper-datalog-volume:
diff --git a/lib/docker_compose/resources/click-house/config.xml b/lib/docker_compose/resources/click-house/config.xml
new file mode 100755
index 00000000..5371846c
--- /dev/null
+++ b/lib/docker_compose/resources/click-house/config.xml
@@ -0,0 +1,35 @@
+
+ /var/lib/clickhouse/format_schemas/
+
+ 1
+
+ ch-1S_1K
+ 0.0.0.0
+ 8123
+ 9000
+
+
+ users.xml
+
+
+ /tmp/lib/clickhouse/access/
+
+
+
+ /clickhouse/task_queue/ddl
+
+
+
+ zookeeper
+ 9181
+
+
+
+ /metrics
+ 9363
+ true
+ true
+ true
+ true
+
+
\ No newline at end of file
diff --git a/lib/docker_compose/resources/click-house/keeper-config.xml b/lib/docker_compose/resources/click-house/keeper-config.xml
new file mode 100755
index 00000000..5ea09bed
--- /dev/null
+++ b/lib/docker_compose/resources/click-house/keeper-config.xml
@@ -0,0 +1,28 @@
+
+
+ information
+ /tmp/log/clickhouse-keeper/clickhouse-keeper.log
+ /tmp/log/clickhouse-keeper/clickhouse-keeper.err.log
+ 1000M
+ 3
+
+ 0.0.0.0
+
+ 9181
+ 1
+ /tmp/lib/clickhouse/coordination/log
+ /tmp/lib/clickhouse/coordination/snapshots
+
+ 10000
+ 30000
+ information
+
+
+
+ 1
+ localhost
+ 9234
+
+
+
+
\ No newline at end of file
diff --git a/lib/docker_compose/resources/click-house/schemas/errors.proto b/lib/docker_compose/resources/click-house/schemas/errors.proto
new file mode 100755
index 00000000..4a56e0ab
--- /dev/null
+++ b/lib/docker_compose/resources/click-house/schemas/errors.proto
@@ -0,0 +1,23 @@
+syntax = "proto3";
+
+package com.browserup.common.metric.dto;
+
+message Error {
+ int64 start_ts = 1;
+ string exception_name = 2;
+ string hostname = 3;
+ uint32 iteration = 4;
+ string region = 5;
+ string profile = 6;
+ string image = 7;
+ string step = 8;
+ uint32 vu_id = 9;
+ uint32 run_id = 10;
+ uint32 workspace_id = 11;
+ uint32 scenario_id = 12;
+ uint32 profile_id = 13;
+}
+
+message ErrorsList {
+ repeated Error errors = 1;
+}
diff --git a/lib/docker_compose/resources/click-house/schemas/run_infos.proto b/lib/docker_compose/resources/click-house/schemas/run_infos.proto
new file mode 100755
index 00000000..6e84bfe6
--- /dev/null
+++ b/lib/docker_compose/resources/click-house/schemas/run_infos.proto
@@ -0,0 +1,15 @@
+syntax = "proto3";
+
+package com.browserup.common.metric.dto;
+
+message Run_info {
+ int64 start_ts = 1;
+ int64 end_ts = 2;
+ string region = 3;
+ uint32 run_id = 4;
+ uint32 workspace_id = 5;
+}
+
+message Run_infosList {
+ repeated Run_info run_infos = 1;
+}
diff --git a/lib/docker_compose/resources/click-house/schemas/steps.proto b/lib/docker_compose/resources/click-house/schemas/steps.proto
new file mode 100755
index 00000000..897b4ab5
--- /dev/null
+++ b/lib/docker_compose/resources/click-house/schemas/steps.proto
@@ -0,0 +1,38 @@
+syntax = "proto3";
+
+package com.browserup.common.metric.dto;
+
+message Step {
+ int64 start_ts = 1;
+ int64 end_ts = 2;
+ string href = 3;
+ string step = 4;
+ string title = 5;
+ int32 dom_interactive_ms = 6;
+ int32 dns_ms = 7;
+ int32 duration_ms = 8;
+ int32 first_input_delay_ms = 9;
+ int32 first_contentful_paint_ms = 10;
+ int32 first_contentful_paint_size = 11;
+ int32 first_paint_ms = 12;
+ int32 largest_contentful_paint_ms = 13;
+ int32 largest_contentful_paint_size = 14;
+ int32 on_content_load_ms = 15;
+ int32 on_load_ms = 16;
+ int32 ssl_ms = 17;
+ int32 time_to_first_byte_ms = 18;
+ double cumulative_layout_shift = 19;
+ uint32 iteration = 20;
+ string region = 21;
+ string profile = 22;
+ string image = 23;
+ uint32 vu_id = 24;
+ uint32 run_id = 25;
+ uint32 workspace_id = 26;
+ uint32 scenario_id = 27;
+ uint32 profile_id = 28;
+}
+
+message StepsList {
+ repeated Step steps = 1;
+}
diff --git a/lib/docker_compose/resources/click-house/schemas/system_stats.proto b/lib/docker_compose/resources/click-house/schemas/system_stats.proto
new file mode 100755
index 00000000..4838d3fa
--- /dev/null
+++ b/lib/docker_compose/resources/click-house/schemas/system_stats.proto
@@ -0,0 +1,26 @@
+syntax = "proto3";
+
+package com.browserup.common.metric.dto;
+
+message System_stat {
+ int64 start_ts = 1;
+ int32 bytes_received = 2;
+ int32 bytes_sent = 3;
+ uint32 cpu_total_usage = 4;
+ uint32 dropped_packets_received = 5;
+ uint32 dropped_packets_sent = 6;
+ uint32 memory_percentage = 7;
+ uint32 memory_usage = 8;
+ int64 host_available_memory = 9;
+ uint32 packets_received = 10;
+ uint32 packets_sent = 11;
+ uint32 run_id = 12;
+ uint32 workspace_id = 13;
+ uint32 scenario_id = 14;
+ string region = 15;
+ string host = 16;
+}
+
+message System_statsList {
+ repeated System_stat system_stats = 1;
+}
diff --git a/lib/docker_compose/resources/click-house/schemas/urls.proto b/lib/docker_compose/resources/click-house/schemas/urls.proto
new file mode 100755
index 00000000..4006b76f
--- /dev/null
+++ b/lib/docker_compose/resources/click-house/schemas/urls.proto
@@ -0,0 +1,56 @@
+syntax = "proto3";
+
+package com.browserup.common.metric.dto;
+
+message Url {
+ string url = 1;
+ int64 start_ts = 2;
+ int64 end_ts = 3;
+ uint32 run_id = 4;
+ string profile = 5;
+ string step = 6;
+ string region = 7;
+ string image = 8;
+ uint32 vu_id = 9;
+ string trace_id = 10;
+ string trace_parent_id = 11;
+ uint32 workspace_id = 12;
+ uint32 scenario_id = 13;
+ uint32 profile_id = 14;
+ int32 status_code = 15;
+ string method = 16;
+ string content_type = 17;
+ int32 ssl_ms = 18;
+ int32 blocked_ms = 19;
+ int32 connect_ms = 20;
+ int32 dns_ms = 21;
+ int32 wait_ms = 22;
+ int32 receive_ms = 23;
+ int32 response_ms = 24;
+ int32 send_ms = 25;
+ int32 ws_messages_rcvd = 26;
+ int32 ws_messages_sent = 27;
+ int32 ws_bytes_sent = 28;
+ int32 ws_bytes_rcvd = 29;
+ int32 video_buffered_percent = 30;
+ int32 video_stall_count = 31;
+ int32 video_decoded_byte_count = 32;
+ int32 video_waiting_count = 33;
+ int32 video_error_count = 34;
+ int32 video_dropped_frames = 35;
+ int32 video_total_frames = 36;
+ int32 video_audio_bytes_decoded = 37;
+ int32 bytes_received_size = 38;
+ int32 bytes_sent_size = 39;
+ int32 request_body_size = 40;
+ int32 response_body_size = 41;
+ int32 request_headers_size = 42;
+ int32 response_headers_size = 43;
+ string content = 44;
+ string headers = 45;
+ uint32 iteration = 46;
+}
+
+message UrlsList {
+ repeated Url urls = 1;
+}
diff --git a/lib/docker_compose/resources/click-house/schemas/verifications.proto b/lib/docker_compose/resources/click-house/schemas/verifications.proto
new file mode 100755
index 00000000..202e5b6d
--- /dev/null
+++ b/lib/docker_compose/resources/click-house/schemas/verifications.proto
@@ -0,0 +1,18 @@
+syntax = "proto3";
+
+package com.browserup.common.metric.dto;
+
+message Verification {
+ uint32 workspace_id = 1;
+ uint32 scenario_id = 2;
+ uint32 run_id = 3;
+ string name = 4;
+ uint32 result = 5;
+ string type = 6;
+ string scenario_name = 7;
+ int64 timestamp = 8;
+}
+
+message VerificationsList {
+ repeated Verification verifications = 1;
+}
diff --git a/lib/docker_compose/resources/click-house/schemas/vu_logs.proto b/lib/docker_compose/resources/click-house/schemas/vu_logs.proto
new file mode 100755
index 00000000..45b248f1
--- /dev/null
+++ b/lib/docker_compose/resources/click-house/schemas/vu_logs.proto
@@ -0,0 +1,24 @@
+syntax = "proto3";
+
+package com.browserup.common.metric.dto;
+
+message Vu_log {
+ int64 start_ts = 1;
+ int64 end_ts = 2;
+ string log_text = 3;
+ uint32 iteration = 4;
+ string status = 5;
+ string region = 6;
+ string profile = 7;
+ string image = 8;
+ string step = 9;
+ uint32 vu_id = 10;
+ uint32 run_id = 11;
+ uint32 workspace_id = 12;
+ uint32 scenario_id = 13;
+ uint32 profile_id = 14;
+}
+
+message Vu_logsList {
+ repeated Vu_log vu_logs = 1;
+}
diff --git a/lib/docker_compose/resources/click-house/schemas/vu_statuses.proto b/lib/docker_compose/resources/click-house/schemas/vu_statuses.proto
new file mode 100755
index 00000000..9e8b01fe
--- /dev/null
+++ b/lib/docker_compose/resources/click-house/schemas/vu_statuses.proto
@@ -0,0 +1,23 @@
+syntax = "proto3";
+
+package com.browserup.common.metric.dto;
+
+message Vu_status {
+ int64 start_ts = 1;
+ int64 end_ts = 2;
+ uint32 iteration = 3;
+ string status = 4;
+ string region = 5;
+ string profile = 6;
+ string image = 7;
+ string step = 8;
+ uint32 vu_id = 9;
+ uint32 run_id = 10;
+ uint32 workspace_id = 11;
+ uint32 scenario_id = 12;
+ uint32 profile_id = 13;
+}
+
+message Vu_statusesList {
+ repeated Vu_status vu_statuses = 1;
+}
diff --git a/lib/docker_compose/resources/click-house/users.xml b/lib/docker_compose/resources/click-house/users.xml
new file mode 100755
index 00000000..2a4948fa
--- /dev/null
+++ b/lib/docker_compose/resources/click-house/users.xml
@@ -0,0 +1,38 @@
+
+
+
+
+ 10000000000
+ 10000000000
+ 0
+ in_order
+ 1
+
+
+
+
+ 1
+ default
+
+ ::/0
+
+ default
+ 1
+ 1
+ 1
+ 1
+
+
+
+
+
+ 3600
+ 0
+ 0
+ 0
+ 0
+ 0
+
+
+
+
\ No newline at end of file
diff --git a/lib/docker_compose/resources/fluent-bit.conf b/lib/docker_compose/resources/fluent-bit.conf
deleted file mode 100644
index 80702adb..00000000
--- a/lib/docker_compose/resources/fluent-bit.conf
+++ /dev/null
@@ -1,12 +0,0 @@
-[INPUT]
- Name forward
- Listen 0.0.0.0
- Port 24224
-
-[Output]
- name loki
- match *
- host loki
- port 3100
- labels container_name=$container_name
- line_format json
diff --git a/lib/docker_compose/resources/fluent-bit/fluent-bit.conf b/lib/docker_compose/resources/fluent-bit/fluent-bit.conf
new file mode 100644
index 00000000..3a775bb1
--- /dev/null
+++ b/lib/docker_compose/resources/fluent-bit/fluent-bit.conf
@@ -0,0 +1,36 @@
+[SERVICE]
+ HTTP_Server On
+ HTTP_Listen 0.0.0.0
+ HTTP_PORT 2020
+ Hot_Reload On
+ Parsers_File /fluent-bit/etc/parsers.conf
+
+[FILTER]
+ Name modify
+ Match user_container_log
+ Condition Key_value_matches log ^browserup_user_log.*
+ Add log_type user
+
+[FILTER]
+ Name modify
+ Match user_container_log
+ Condition Key_Value_Does_Not_Match log ^browserup_user_log.*
+ Add log_type system
+
+[INPUT]
+ name http
+ listen 0.0.0.0
+ port 8888
+
+[INPUT]
+ Name forward
+ Listen 0.0.0.0
+ Port 8889
+
+[Output]
+ name loki
+ match *
+ host loki
+ port 3100
+ labels container_name=$container_name $log_type
+ line_format json
\ No newline at end of file
diff --git a/lib/docker_compose/resources/grafana-cadvisor-dashboard.json b/lib/docker_compose/resources/grafana-cadvisor-dashboard.json
new file mode 100644
index 00000000..f7a09ad4
--- /dev/null
+++ b/lib/docker_compose/resources/grafana-cadvisor-dashboard.json
@@ -0,0 +1,835 @@
+{
+ "__inputs": [
+ {
+ "name": "DS_PROMETHEUS",
+ "label": "Prometheus",
+ "description": "Prometheus as the datasource is obligatory",
+ "type": "datasource",
+ "pluginId": "prometheus",
+ "pluginName": "Prometheus"
+ }
+ ],
+ "__requires": [
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "7.4.5"
+ },
+ {
+ "type": "panel",
+ "id": "graph",
+ "name": "Graph",
+ "version": ""
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "1.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "table",
+ "name": "Table",
+ "version": ""
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "gnetId": 14282,
+ "graphTooltip": 0,
+ "id": null,
+ "iteration": 1617715580880,
+ "links": [],
+ "panels": [
+ {
+ "collapsed": false,
+ "datasource": "$DS_PROMETHEUS",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 8,
+ "panels": [],
+ "title": "CPU",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 24,
+ "x": 0,
+ "y": 1
+ },
+ "hiddenSeries": false,
+ "id": 15,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.4.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(container_cpu_usage_seconds_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name) *100",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "CPU Usage",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:606",
+ "format": "percent",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:607",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "collapsed": false,
+ "datasource": "$DS_PROMETHEUS",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 8
+ },
+ "id": 11,
+ "panels": [],
+ "title": "Memory",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 9
+ },
+ "hiddenSeries": false,
+ "id": 9,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.4.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(container_memory_rss{instance=~\"$host\",name=~\"$container\",name=~\".+\"}) by (name)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Memory Usage",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:606",
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:607",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 9
+ },
+ "hiddenSeries": false,
+ "id": 14,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.4.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(container_memory_cache{instance=~\"$host\",name=~\"$container\",name=~\".+\"}) by (name)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Memory Cached",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:606",
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:607",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "collapsed": false,
+ "datasource": "$DS_PROMETHEUS",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 17
+ },
+ "id": 2,
+ "panels": [],
+ "title": "Network",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 18
+ },
+ "hiddenSeries": false,
+ "id": 4,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sideWidth": null,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.4.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(container_network_receive_bytes_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Received Network Traffic",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:674",
+ "format": "Bps",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:675",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 18
+ },
+ "hiddenSeries": false,
+ "id": 6,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.4.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(container_network_transmit_bytes_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name)",
+ "interval": "",
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Sent Network Traffic",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:832",
+ "format": "Bps",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:833",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "collapsed": false,
+ "datasource": "$DS_PROMETHEUS",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 26
+ },
+ "id": 19,
+ "panels": [],
+ "title": "Misc",
+ "type": "row"
+ },
+ {
+ "datasource": "$DS_PROMETHEUS",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "align": null,
+ "filterable": false
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "id"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 260
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Running"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "d"
+ },
+ {
+ "id": "decimals",
+ "value": 1
+ },
+ {
+ "id": "custom.displayMode",
+ "value": "color-text"
+ },
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "dark-green",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 24,
+ "x": 0,
+ "y": 27
+ },
+ "id": 17,
+ "options": {
+ "showHeader": true,
+ "sortBy": []
+ },
+ "pluginVersion": "7.4.5",
+ "targets": [
+ {
+ "expr": "(time() - container_start_time_seconds{instance=~\"$host\",name=~\"$container\",name=~\".+\"})/86400",
+ "format": "table",
+ "instant": true,
+ "interval": "",
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Containers Info",
+ "transformations": [
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "names": [
+ "container_label_com_docker_compose_project",
+ "container_label_com_docker_compose_project_working_dir",
+ "image",
+ "instance",
+ "name",
+ "Value",
+ "container_label_com_docker_compose_service"
+ ]
+ }
+ }
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {},
+ "indexByName": {},
+ "renameByName": {
+ "Value": "Running",
+ "container_label_com_docker_compose_project": "Label",
+ "container_label_com_docker_compose_project_working_dir": "Working dir",
+ "container_label_com_docker_compose_service": "Service",
+ "image": "Registry Image",
+ "instance": "Instance",
+ "name": "Name"
+ }
+ }
+ }
+ ],
+ "type": "table"
+ }
+ ],
+ "schemaVersion": 27,
+ "style": "dark",
+ "tags": [
+ "cadvisor",
+ "docker"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "Prometheus",
+ "value": "Prometheus"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "Datasource",
+ "multi": false,
+ "name": "DS_PROMETHEUS",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "allValue": ".*",
+ "current": {},
+ "datasource": "$DS_PROMETHEUS",
+ "definition": "label_values({__name__=~\"container.*\"},instance)",
+ "description": null,
+ "error": null,
+ "hide": 0,
+ "includeAll": true,
+ "label": "Host",
+ "multi": false,
+ "name": "host",
+ "options": [],
+ "query": {
+ "query": "label_values({__name__=~\"container.*\"},instance)",
+ "refId": "Prometheus-host-Variable-Query"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 5,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".*",
+ "current": {},
+ "datasource": "$DS_PROMETHEUS",
+ "definition": "label_values({__name__=~\"container.*\", instance=~\"$host\"},name)",
+ "description": null,
+ "error": null,
+ "hide": 0,
+ "includeAll": true,
+ "label": "Container",
+ "multi": false,
+ "name": "container",
+ "options": [],
+ "query": {
+ "query": "label_values({__name__=~\"container.*\", instance=~\"$host\"},name)",
+ "refId": "Prometheus-container-Variable-Query"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {},
+ "timezone": "",
+ "title": "Cadvisor exporter",
+ "uid": "pMEd7m0Mz",
+ "version": 1,
+ "description": "Simple exporter for cadvisor only"
+}
\ No newline at end of file
diff --git a/lib/docker_compose/resources/grafana-clickhouse-dashboard.json b/lib/docker_compose/resources/grafana-clickhouse-dashboard.json
new file mode 100644
index 00000000..5c761ef6
--- /dev/null
+++ b/lib/docker_compose/resources/grafana-clickhouse-dashboard.json
@@ -0,0 +1,11260 @@
+{
+ "__inputs": [
+ {
+ "name": "DS_PROMETHEUS",
+ "label": "Prometheus",
+ "description": "",
+ "type": "datasource",
+ "pluginId": "prometheus",
+ "pluginName": "Prometheus"
+ }
+ ],
+ "__requires": [
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "7.5.2"
+ },
+ {
+ "type": "panel",
+ "id": "graph",
+ "name": "Graph",
+ "version": ""
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "1.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "table",
+ "name": "Table",
+ "version": ""
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ },
+ {
+ "datasource": "$DS_PROMETHEUS",
+ "enable": true,
+ "expr": "resets(ClickHouseAsyncMetrics_Uptime{instance=~\"$instance\"}[$__rate_interval])",
+ "hide": false,
+ "iconColor": "rgba(255, 96, 96, 1)",
+ "name": "Restarts",
+ "showIn": 0,
+ "step": "60s",
+ "tagKeys": "instance",
+ "titleFormat": "Restart",
+ "useValueForTime": false
+ }
+ ]
+ },
+ "description": "ClickHouse internal exporter metrics",
+ "editable": true,
+ "gnetId": 14192,
+ "graphTooltip": 1,
+ "id": null,
+ "iteration": 1617958425076,
+ "links": [
+ {
+ "asDropdown": false,
+ "icon": "external link",
+ "includeVars": false,
+ "keepTime": false,
+ "tags": [],
+ "targetBlank": true,
+ "title": "ClickHouse monitoring docs",
+ "tooltip": "",
+ "type": "link",
+ "url": "https://clickhouse.tech/docs/en/operations/monitoring/"
+ }
+ ],
+ "panels": [
+ {
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": null,
+ "filterable": false
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "uptime"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "s"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 14,
+ "options": {
+ "frameIndex": 1,
+ "showHeader": true
+ },
+ "pluginVersion": "7.5.2",
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_VersionInteger",
+ "format": "table",
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_Revision",
+ "format": "table",
+ "hide": false,
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "refId": "B"
+ },
+ {
+ "exemplar": false,
+ "expr": "ClickHouseAsyncMetrics_NumberOfTables",
+ "format": "table",
+ "hide": false,
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "refId": "C"
+ },
+ {
+ "exemplar": false,
+ "expr": "ClickHouseAsyncMetrics_Uptime",
+ "format": "table",
+ "hide": false,
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "refId": "D"
+ },
+ {
+ "exemplar": false,
+ "expr": "ClickHouseAsyncMetrics_NumberOfDatabases",
+ "format": "table",
+ "hide": false,
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "refId": "E"
+ }
+ ],
+ "title": "Nodes",
+ "transformations": [
+ {
+ "id": "seriesToColumns",
+ "options": {
+ "byField": "instance"
+ }
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {
+ "Time 1": true,
+ "Time 10": true,
+ "Time 2": true,
+ "Time 3": true,
+ "Time 4": true,
+ "Time 5": true,
+ "Time 6": true,
+ "Time 7": true,
+ "Time 8": true,
+ "Time 9": true,
+ "Value #A 2": true,
+ "Value #B 2": true,
+ "Value #C 2": true,
+ "Value #D 2": true,
+ "Value #E 2": true,
+ "__name__ 1": true,
+ "__name__ 10": true,
+ "__name__ 2": true,
+ "__name__ 3": true,
+ "__name__ 4": true,
+ "__name__ 5": true,
+ "__name__ 6": true,
+ "__name__ 7": true,
+ "__name__ 8": true,
+ "__name__ 9": true,
+ "job 1": true,
+ "job 10": true,
+ "job 2": true,
+ "job 3": true,
+ "job 4": true,
+ "job 5": true,
+ "job 6": true,
+ "job 7": true,
+ "job 8": true,
+ "job 9": true
+ },
+ "indexByName": {
+ "Time 1": 6,
+ "Time 10": 37,
+ "Time 2": 9,
+ "Time 3": 13,
+ "Time 4": 16,
+ "Time 5": 20,
+ "Time 6": 23,
+ "Time 7": 27,
+ "Time 8": 30,
+ "Time 9": 34,
+ "Value #A 1": 1,
+ "Value #A 2": 12,
+ "Value #B 1": 2,
+ "Value #B 2": 19,
+ "Value #C 1": 3,
+ "Value #C 2": 26,
+ "Value #D 1": 5,
+ "Value #D 2": 33,
+ "Value #E 1": 4,
+ "Value #E 2": 40,
+ "__name__ 1": 7,
+ "__name__ 10": 38,
+ "__name__ 2": 10,
+ "__name__ 3": 14,
+ "__name__ 4": 17,
+ "__name__ 5": 21,
+ "__name__ 6": 24,
+ "__name__ 7": 28,
+ "__name__ 8": 31,
+ "__name__ 9": 35,
+ "instance": 0,
+ "job 1": 8,
+ "job 10": 39,
+ "job 2": 11,
+ "job 3": 15,
+ "job 4": 18,
+ "job 5": 22,
+ "job 6": 25,
+ "job 7": 29,
+ "job 8": 32,
+ "job 9": 36
+ },
+ "renameByName": {
+ "Value #A 1": "version",
+ "Value #B 1": "revision",
+ "Value #C 1": "tables",
+ "Value #D 1": "uptime",
+ "Value #E 1": "databases"
+ }
+ }
+ }
+ ],
+ "transparent": true,
+ "type": "table"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Total amount of memory (bytes) allocated by the server.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 5,
+ "w": 12,
+ "x": 0,
+ "y": 8
+ },
+ "hiddenSeries": false,
+ "id": 29,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_MemoryTracking",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Memory",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "bytes",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of connections to TCP server (clients with native interface), also included server-server distributed query connections. \nNumber of connections to HTTP server",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 5,
+ "w": 12,
+ "x": 12,
+ "y": 8
+ },
+ "hiddenSeries": false,
+ "id": 73,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_TCPConnection",
+ "interval": "",
+ "legendFormat": "tcp - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_HTTPConnection",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "http - {{instance}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Connections",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of times the number of query processing threads was lowered due to slow reads.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 5,
+ "w": 12,
+ "x": 0,
+ "y": 13
+ },
+ "hiddenSeries": false,
+ "id": 72,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBackoff{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ReadBackoff{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Read backoff",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of reads from a file that were slow. This indicate system overload. Thresholds are controlled by read_backoff_* settings.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 5,
+ "w": 12,
+ "x": 12,
+ "y": 13
+ },
+ "hiddenSeries": false,
+ "id": 71,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_SlowRead{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_SlowRead{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Slow reads",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "collapsed": true,
+ "datasource": "$DS_PROMETHEUS",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 18
+ },
+ "id": 12,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 19
+ },
+ "hiddenSeries": false,
+ "id": 2,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_Query{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_Query{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Queries",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Same as Queries, but only for SELECT queries.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 19
+ },
+ "hiddenSeries": false,
+ "id": 3,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectQuery{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_SelectQuery{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "SELECT queries",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Same as Queries, but only for INSERT queries.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 27
+ },
+ "hiddenSeries": false,
+ "id": 4,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_InsertQuery{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_InsertQuery{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "INSERT queries",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Same as Failed queries, but only for SELECT queries.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 27
+ },
+ "hiddenSeries": false,
+ "id": 6,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_FailedSelectQuery{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_FailedSelectQuery{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Failed SELECT queries",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of failed queries.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 35
+ },
+ "hiddenSeries": false,
+ "id": 5,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_FailedQuery{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_FailedQuery{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Failed queries",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of times when memory limit exceeded for query.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 35
+ },
+ "hiddenSeries": false,
+ "id": 85,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_QueryMemoryLimitExceeded{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_QueryMemoryLimitExceeded{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Queries memory limit",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of queries that are stopped and waiting due to 'priority' setting.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 43
+ },
+ "hiddenSeries": false,
+ "id": 86,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseMetrics_QueryPreempted{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseMetrics_QueryPreempted{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Query preemted",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Same as Failed queries, but only for INSERT queries.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 43
+ },
+ "hiddenSeries": false,
+ "id": 7,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_FailedInsertQuery{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_FailedInsertQuery{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Failed INSERT queries",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Avg queries latencies",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 51
+ },
+ "hiddenSeries": false,
+ "id": 8,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "increase(ClickHouseProfileEvents_QueryTimeMicroseconds{instance=~\"$instance\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_Query{instance=~\"$instance\"}[$__rate_interval])",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Query latencies (avg)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "µs",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": 1
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Avg SELECT queries latencies",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 51
+ },
+ "hiddenSeries": false,
+ "id": 9,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "increase(ClickHouseProfileEvents_SelectQueryTimeMicroseconds{instance=~\"$instance\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_SelectQuery{instance=~\"$instance\"}[$__rate_interval])",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "SELECT query latencies (avg)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "µs",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Avg INSERT queries latencies",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 59
+ },
+ "hiddenSeries": false,
+ "id": 10,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "increase(ClickHouseProfileEvents_InsertQueryTimeMicroseconds{instance=~\"$instance\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_InsertQuery{instance=~\"$instance\"}[$__rate_interval])",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "INSERT query latencies (avg)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "µs",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "Queries",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": "$DS_PROMETHEUS",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 19
+ },
+ "id": 64,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of rows INSERTed to all tables.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 30
+ },
+ "hiddenSeries": false,
+ "id": 62,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_InsertedRows{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_InsertedRows{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Inserted rows",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of bytes (uncompressed; for columns as they stored in memory) INSERTed to all tables.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 30
+ },
+ "hiddenSeries": false,
+ "id": 65,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_InsertedBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_InsertedBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Inserted bytes",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "binBps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of times the INSERT of a block to a MergeTree table was throttled due to high number of active data parts for partition.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 38
+ },
+ "hiddenSeries": false,
+ "id": 66,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_DelayedInserts{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_DelayedInserts{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Delayed inserts",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of times the INSERT of a block to a MergeTree table was rejected with 'Too many parts' exception due to high number of active data parts for partition.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 38
+ },
+ "hiddenSeries": false,
+ "id": 67,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_RejectedInserts{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_RejectedInserts{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Rejected inserts",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Total number of milliseconds spent while the INSERT of a block to a MergeTree table was throttled due to high number of active data parts for partition.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 46
+ },
+ "hiddenSeries": false,
+ "id": 69,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "increase(ClickHouseProfileEvents_DelayedInsertsMilliseconds{instance=~\"$instance\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_DelayedInserts{instance=~\"$instance\"}[$__rate_interval])",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Delayed inserts blocks (avg)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "µs",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": 1
+ }
+ }
+ ],
+ "title": "Insert",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": "$DS_PROMETHEUS",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 20
+ },
+ "id": 125,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of data parts selected to read from a MergeTree table.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 27
+ },
+ "hiddenSeries": false,
+ "id": 126,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedParts{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_SelectedParts{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Parts",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of (non-adjacent) ranges in all data parts selected to read from a MergeTree table.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 27
+ },
+ "hiddenSeries": false,
+ "id": 127,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedRanges{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_SelectedRanges{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Ranges",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of marks (index granules) selected to read from a MergeTree table.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 35
+ },
+ "hiddenSeries": false,
+ "id": 128,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedMarks{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_SelectedMarks{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Marks",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of rows SELECTed from all tables.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 35
+ },
+ "hiddenSeries": false,
+ "id": 129,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedRows{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_SelectedRows{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Rows",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of bytes (uncompressed; for columns as they stored in memory) SELECTed from all tables.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 43
+ },
+ "hiddenSeries": false,
+ "id": 130,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_SelectedBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Bytes",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "binBps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "Select",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": "$DS_PROMETHEUS",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 21
+ },
+ "id": 17,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of times the 'lseek' function was called.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 33
+ },
+ "hiddenSeries": false,
+ "id": 20,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_Seek{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_Seek{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "lseek",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "ops",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of files opened",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 33
+ },
+ "hiddenSeries": false,
+ "id": 19,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_FileOpen{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_FileOpen{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Open Files",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "ops",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of reads (read/pread) from a file descriptor. Does not include sockets.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 41
+ },
+ "hiddenSeries": false,
+ "id": 22,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorRead{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorRead{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Read from FD",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "ops",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of writes (write/pwrite) to a file descriptor. Does not include sockets.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 41
+ },
+ "hiddenSeries": false,
+ "id": 23,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWrite{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWrite{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Write to FD",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "ops",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of times the read (read/pread) from a file descriptor have failed.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 49
+ },
+ "hiddenSeries": false,
+ "id": 24,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadFailed{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadFailed{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Read from FD failed",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "ops",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of times the read (read/pread) from a file descriptor have failed.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 49
+ },
+ "hiddenSeries": false,
+ "id": 25,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadFailed{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadFailed{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Read from FD failed",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "ops",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of bytes read from file descriptors. If the file is compressed, this will show the compressed data size.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 57
+ },
+ "hiddenSeries": false,
+ "id": 21,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Read bytes from FD",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "binBps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of bytes written to file descriptors. If the file is compressed, this will show compressed data size.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 57
+ },
+ "hiddenSeries": false,
+ "id": 26,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWriteBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWriteBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Write bytes to FD",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "binBps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of bytes (the number of bytes before decompression) read from compressed sources (files, network).",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 65
+ },
+ "hiddenSeries": false,
+ "id": 51,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadCompressedBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ReadCompressedBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Read compressed bytes",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "binBps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of uncompressed bytes (the number of bytes after decompression) read from compressed sources (files, network).",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 65
+ },
+ "hiddenSeries": false,
+ "id": 53,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_CompressedReadBufferBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_CompressedReadBufferBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Read uncompressed bytes",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "binBps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of compressed blocks (the blocks of data that are compressed independent of each other) read from compressed sources (files, network).",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 73
+ },
+ "hiddenSeries": false,
+ "id": 52,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_CompressedReadBufferBlocks{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_CompressedReadBufferBlocks{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Compressed blocks",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "IO",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": "$DS_PROMETHEUS",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 22
+ },
+ "id": 56,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of Replicated tables that are currently in readonly state due to re-initialization after ZooKeeper session loss or due to startup without ZooKeeper configured.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 31
+ },
+ "hiddenSeries": false,
+ "id": 74,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_ReadonlyReplica",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Readonly replica",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of times a data part was failed to download from replica of a ReplicatedMergeTree table.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 31
+ },
+ "hiddenSeries": false,
+ "id": 58,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartFailedFetches{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ReplicatedPartFailedFetches{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Replicated part failed fetches",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of times a data part was downloaded from replica of a ReplicatedMergeTree table.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 39
+ },
+ "hiddenSeries": false,
+ "id": 57,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartFetches{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ReplicatedPartFetches{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Replicated part fetches",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of times we prefer to download already merged part from replica of ReplicatedMergeTree table instead of performing a merge ourself (usually we prefer doing a merge ourself to save network traffic). This happens when we have not all source parts to perform a merge or when the data part is old enough.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 39
+ },
+ "hiddenSeries": false,
+ "id": 60,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartFetchesOfMerged{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ReplicatedPartFetchesOfMerged{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Replicated part fetches or merges",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of times data parts of ReplicatedMergeTree tables were successfully merged.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 47
+ },
+ "hiddenSeries": false,
+ "id": 59,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartMerges{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ReplicatedPartMerges{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Replicated part merges",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of times a data part that we wanted doesn't exist on any replica (even on replicas that are offline right now). That data parts are definitely lost. This is normal due to asynchronous replication (if quorum inserts were not enabled), when the replica on which the data part was written was failed and when it became online after fail it doesn't contain that data part.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 47
+ },
+ "hiddenSeries": false,
+ "id": 61,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedDataLoss{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ReplicatedDataLoss{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Replicated part data loss",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "Replicas",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": "$DS_PROMETHEUS",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 23
+ },
+ "id": 76,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of launched background merges.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 29
+ },
+ "hiddenSeries": false,
+ "id": 77,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_Merge{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_Merge{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Merge",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Rows read for background merges. This is the number of rows before merge.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 29
+ },
+ "hiddenSeries": false,
+ "id": 78,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_MergedRows{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_MergedRows{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Merged rows",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Uncompressed bytes (for columns as they stored in memory) that was read for background merges. This is the number before merge.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 37
+ },
+ "hiddenSeries": false,
+ "id": 79,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_MergedUncompressedBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_MergedUncompressedBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Merge uncompressed bytes",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "binBps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Avg merge duration",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 37
+ },
+ "hiddenSeries": false,
+ "id": 80,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "increase(ClickHouseProfileEvents_MergesTimeMilliseconds{instance=~\"$instance\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_Merge{instance=~\"$instance\"}[$__rate_interval])",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Merge duration (avg)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "µs",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": 1
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of rows INSERTed to MergeTree tables.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 45
+ },
+ "hiddenSeries": false,
+ "id": 81,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterRows{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_MergeTreeDataWriterRows{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "MergeTree rows",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of blocks INSERTed to MergeTree tables. Each block forms a data part of level zero.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 45
+ },
+ "hiddenSeries": false,
+ "id": 82,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterBlocks{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_MergeTreeDataWriterBlocks{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "MergeTree blocks",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Uncompressed bytes (for columns as they stored in memory) INSERTed to MergeTree tables.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 53
+ },
+ "hiddenSeries": false,
+ "id": 83,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterUncompressedBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_MergeTreeDataWriterUncompressedBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "MergeTree uncompressed bytes",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "binBps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Bytes written to filesystem for data INSERTed to MergeTree tables.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 53
+ },
+ "hiddenSeries": false,
+ "id": 84,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterCompressedBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_MergeTreeDataWriterCompressedBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "MergeTree compressed bytes",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "binBps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Disk space reserved for currently running background merges. It is slightly more than the total size of currently merging parts.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 61
+ },
+ "hiddenSeries": false,
+ "id": 123,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_DiskSpaceReservedForMerge",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Reserved space",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "bytes",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "Merge",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": "$DS_PROMETHEUS",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 24
+ },
+ "id": 92,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 26
+ },
+ "hiddenSeries": false,
+ "id": 102,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_UncompressedCacheHits{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_UncompressedCacheHits{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Uncompressed Cache Hits",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 26
+ },
+ "hiddenSeries": false,
+ "id": 103,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_UncompressedCacheMisses{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_UncompressedCacheMisses{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Uncompressed Cache Misses",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 34
+ },
+ "hiddenSeries": false,
+ "id": 104,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_MarkCacheHits{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_MarkCacheHits{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Mark Cache Hits",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 34
+ },
+ "hiddenSeries": false,
+ "id": 105,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_MarkCacheMisses{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_MarkCacheMisses{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Mark Cache Misses",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "Cache",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": "$DS_PROMETHEUS",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 25
+ },
+ "id": 90,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "The part is generating now, it is not in data_parts list.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 27
+ },
+ "hiddenSeries": false,
+ "id": 114,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_PartsTemporary",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Temporary",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "The part is in data_parts, but not used for SELECTs.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 27
+ },
+ "hiddenSeries": false,
+ "id": 115,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_PartsPreCommitted",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Pre commited",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "The part is in data_parts, but not used for SELECTs.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 35
+ },
+ "hiddenSeries": false,
+ "id": 116,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_PartsCommitted",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Commited",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Not active data part, but could be used by only current SELECTs, could be deleted after SELECTs finishes.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 35
+ },
+ "hiddenSeries": false,
+ "id": 117,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_PartsOutdated",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Outdated",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Not active data part with identity refcounter, it is deleting right now by a cleaner.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 43
+ },
+ "hiddenSeries": false,
+ "id": 118,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_PartsDeleting",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Deleting",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Part was moved to another disk and should be deleted in own destructor.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 43
+ },
+ "hiddenSeries": false,
+ "id": 119,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_PartsDeleteOnDestroy",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Delete on destroy",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Wide parts.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 51
+ },
+ "hiddenSeries": false,
+ "id": 120,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_PartsWide",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Wide",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Compact parts.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 51
+ },
+ "hiddenSeries": false,
+ "id": 122,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_PartsCompact",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Compact",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "In-memory parts.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 59
+ },
+ "hiddenSeries": false,
+ "id": 121,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_PartsInMemory",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "In-memory",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "Parts",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": "$DS_PROMETHEUS",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 26
+ },
+ "id": 94,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of connections to remote servers sending data that was INSERTed into Distributed tables. Both synchronous and asynchronous mode.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 25
+ },
+ "hiddenSeries": false,
+ "id": 112,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_DistributedSend",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Send",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of pending files to process for asynchronous insertion into Distributed tables. Number of files for every shard is summed.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 25
+ },
+ "hiddenSeries": false,
+ "id": 113,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_DistributedFilesToInsert",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Files to insert",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of times the INSERT of a block to a Distributed table was throttled due to high number of pending bytes.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 33
+ },
+ "hiddenSeries": false,
+ "id": 106,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedDelayedInserts{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_DistributedDelayedInserts{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Distributed delayed inserts",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of times the INSERT of a block to a Distributed table was rejected with 'Too many bytes' exception due to high number of pending bytes.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 33
+ },
+ "hiddenSeries": false,
+ "id": 107,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedRejectedInserts{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_DistributedRejectedInserts{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Distributed rejected inserts",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Total number of milliseconds spent while the INSERT of a block to a Distributed table was throttled due to high number of pending bytes.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 41
+ },
+ "hiddenSeries": false,
+ "id": 108,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "increase(ClickHouseProfileEvents_DistributedDelayedInsertsMilliseconds{instance=~\"$instance\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_DistributedDelayedInserts{instance=~\"$instance\"}[$__rate_interval])",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Distributed delayed inserts blocks (avg)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "µs",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": 1
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Total count when distributed connection fails with retry",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 41
+ },
+ "hiddenSeries": false,
+ "id": 109,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedConnectionFailTry{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_DistributedConnectionFailTry{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Distributed connection fail try",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Total count when distributed connection fails after all retries finished",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 49
+ },
+ "hiddenSeries": false,
+ "id": 110,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedConnectionFailAtAll{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_DistributedConnectionFailAtAll{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Distributed connection fail at all",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 49
+ },
+ "hiddenSeries": false,
+ "id": 111,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedSyncInsertionTimeoutExceeded{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_DistributedSyncInsertionTimeoutExceeded{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Distributed sync insertation timeout",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "Distributed",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": "$DS_PROMETHEUS",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 27
+ },
+ "id": 88,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of active tasks in BackgroundProcessingPool (merges, mutations, or replication queue bookkeeping)",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 28
+ },
+ "hiddenSeries": false,
+ "id": 95,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_BackgroundPoolTask",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "BackgroundPool task",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of active tasks in BackgroundFetchesPool",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 28
+ },
+ "hiddenSeries": false,
+ "id": 96,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_BackgroundFetchesPoolTask",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "BackgroundFetchesPool task",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of active tasks in BackgroundProcessingPool for moves",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 36
+ },
+ "hiddenSeries": false,
+ "id": 97,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_BackgroundMovePoolTask",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "BackgroundMovePool task",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 36
+ },
+ "hiddenSeries": false,
+ "id": 98,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_BackgroundSchedulePoolTask",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "BackgroundSchedulePool task",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of active tasks in BackgroundBufferFlushSchedulePool. This pool is used for periodic Buffer flushes",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 44
+ },
+ "hiddenSeries": false,
+ "id": 99,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_BackgroundBufferFlushSchedulePoolTask",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "BackgroundBufferFlushSchedulePool task",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of active tasks in BackgroundDistributedSchedulePool. This pool is used for distributed sends that is done in background.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 44
+ },
+ "hiddenSeries": false,
+ "id": 100,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_BackgroundDistributedSchedulePoolTask",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "BackgroundDistributedSchedulePool task",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of active tasks in BackgroundProcessingPool for message streaming",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 52
+ },
+ "hiddenSeries": false,
+ "id": 101,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_BackgroundMessageBrokerSchedulePoolTask",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "BackgroundMessageBrokerSchedulePool task",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "Background pool",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": "$DS_PROMETHEUS",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 28
+ },
+ "id": 28,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 29
+ },
+ "hiddenSeries": false,
+ "id": 30,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperInit{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ZooKeeperInit{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Init",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 29
+ },
+ "hiddenSeries": false,
+ "id": 31,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperTransactions{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ZooKeeperTransactions{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Transactions",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 37
+ },
+ "hiddenSeries": false,
+ "id": 32,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperList{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ZooKeeperList{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "List",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 37
+ },
+ "hiddenSeries": false,
+ "id": 33,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperCreate{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ZooKeeperCreate{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Create",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 45
+ },
+ "hiddenSeries": false,
+ "id": 34,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperRemove{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ZooKeeperRemove{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Remove",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 45
+ },
+ "hiddenSeries": false,
+ "id": 35,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperExists{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ZooKeeperExists{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Exists",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 53
+ },
+ "hiddenSeries": false,
+ "id": 36,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperGet{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ZooKeeperGet{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Get",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 53
+ },
+ "hiddenSeries": false,
+ "id": 37,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperSet{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ZooKeeperSet{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Set",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 61
+ },
+ "hiddenSeries": false,
+ "id": 38,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperMulti{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ZooKeeperMulti{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Multi",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 61
+ },
+ "hiddenSeries": false,
+ "id": 39,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperCheck{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ZooKeeperCheck{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Check",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 69
+ },
+ "hiddenSeries": false,
+ "id": 40,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperClose{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ZooKeeperClose{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Close",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 69
+ },
+ "hiddenSeries": false,
+ "id": 41,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperWatchResponse{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ZooKeeperWatchResponse{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Watch response",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "reqps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 77
+ },
+ "hiddenSeries": false,
+ "id": 42,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperUserExceptions{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ZooKeeperUserExceptions{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "User Exceptions",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 77
+ },
+ "hiddenSeries": false,
+ "id": 43,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperHardwareExceptions{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ZooKeeperHardwareExceptions{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Hardware Exceptions",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 85
+ },
+ "hiddenSeries": false,
+ "id": 46,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperBytesReceived{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ZooKeeperBytesReceived{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Bytes received",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "binBps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 85
+ },
+ "hiddenSeries": false,
+ "id": 45,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperBytesSent{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_ZooKeeperBytesSent{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Bytes sent",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "binBps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of sessions (connections) to ZooKeeper. Should be no more than one, because using more than one connection to ZooKeeper may lead to bugs due to lack of linearizability (stale reads) that ZooKeeper consistency model allows.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 93
+ },
+ "hiddenSeries": false,
+ "id": 48,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_ZooKeeperSession",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Sessions",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of ephemeral nodes hold in ZooKeeper.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 93
+ },
+ "hiddenSeries": false,
+ "id": 47,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_EphemeralNode",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Ephemeral Node",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of watches (event subscriptions) in ZooKeeper.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 101
+ },
+ "hiddenSeries": false,
+ "id": 50,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_ZooKeeperWatch",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Watches",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of requests to ZooKeeper in fly.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 101
+ },
+ "hiddenSeries": false,
+ "id": 49,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "ClickHouseMetrics_ZooKeeperRequest",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Requests",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$DS_PROMETHEUS",
+ "description": "Number of times an error happened while trying to remove ephemeral node. This is not an issue, because our implementation of ZooKeeper library guarantee that the session will expire and the node will be removed.",
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 109
+ },
+ "hiddenSeries": false,
+ "id": 70,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "maxDataPoints": 200,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "exemplar": false,
+ "expr": "max_over_time( (irate(ClickHouseProfileEvents_CannotRemoveEphemeralNode{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
+ "interval": "",
+ "legendFormat": "peaks - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "exemplar": false,
+ "expr": "rate(ClickHouseProfileEvents_CannotRemoveEphemeralNode{instance=~\"$instance\"}[$__rate_interval]) * $trends",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "trend - {{instance}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Remove ephemeral node failed",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "transparent": true,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "cps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "Zookeeper",
+ "type": "row"
+ }
+ ],
+ "refresh": "30s",
+ "schemaVersion": 27,
+ "style": "dark",
+ "tags": [
+ "ClickHouse",
+ "DB"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "Prometheus",
+ "value": "Prometheus"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "Datasource",
+ "multi": false,
+ "name": "DS_PROMETHEUS",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "Prometheus",
+ "value": "Prometheus"
+ },
+ "description": null,
+ "error": null,
+ "hide": 0,
+ "includeAll": false,
+ "label": "DataSource",
+ "multi": false,
+ "name": "datasource",
+ "options": [],
+ "query": "prometheus",
+ "queryValue": "",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "allValue": null,
+ "current": {},
+ "datasource": "$DS_PROMETHEUS",
+ "definition": "label_values(ClickHouseAsyncMetrics_Uptime, instance)",
+ "description": null,
+ "error": null,
+ "hide": 0,
+ "includeAll": true,
+ "label": "Node",
+ "multi": true,
+ "name": "instance",
+ "options": [],
+ "query": {
+ "query": "label_values(ClickHouseAsyncMetrics_Uptime, instance)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 5,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {
+ "selected": false,
+ "text": "Yes",
+ "value": "1"
+ },
+ "description": null,
+ "error": null,
+ "hide": 0,
+ "includeAll": false,
+ "label": "Show trends",
+ "multi": false,
+ "name": "trends",
+ "options": [
+ {
+ "selected": true,
+ "text": "Yes",
+ "value": "1"
+ },
+ {
+ "selected": false,
+ "text": "No",
+ "value": "null"
+ }
+ ],
+ "query": "Yes : 1, No : null",
+ "queryValue": "",
+ "skipUrlSync": false,
+ "type": "custom"
+ },
+ {
+ "allValue": null,
+ "current": {
+ "selected": false,
+ "text": "No",
+ "value": "null"
+ },
+ "description": "Be aware of the points limit per timeseries for Grafana. This option may not work for a time range > 24h",
+ "error": null,
+ "hide": 0,
+ "includeAll": false,
+ "label": "Show peaks",
+ "multi": false,
+ "name": "peaks",
+ "options": [
+ {
+ "selected": false,
+ "text": "Yes",
+ "value": "1"
+ },
+ {
+ "selected": true,
+ "text": "No",
+ "value": "null"
+ }
+ ],
+ "query": "Yes : 1, No : null",
+ "queryValue": "",
+ "skipUrlSync": false,
+ "type": "custom"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-3h",
+ "to": "now"
+ },
+ "timepicker": {},
+ "timezone": "",
+ "title": "ClickHouse",
+ "uid": "thEkJB_Mz",
+ "version": 40
+}
\ No newline at end of file
diff --git a/lib/docker_compose/resources/init.sql b/lib/docker_compose/resources/init.sql
deleted file mode 100644
index ba065568..00000000
--- a/lib/docker_compose/resources/init.sql
+++ /dev/null
@@ -1,5 +0,0 @@
-CREATE DATABASE browserup_test;
-
-GRANT ALL PRIVILEGES ON browserup_test.* TO 'browserup'@'%' WITH GRANT OPTION;
-
-FLUSH PRIVILEGES;
diff --git a/lib/docker_compose/resources/nats/nats-server.conf b/lib/docker_compose/resources/nats/nats-server.conf
new file mode 100644
index 00000000..bdf6478e
--- /dev/null
+++ b/lib/docker_compose/resources/nats/nats-server.conf
@@ -0,0 +1,2 @@
+max_payload: 104857600 # 10 MB
+max_pending: 104857600 # 10 MB
\ No newline at end of file
diff --git a/lib/docker_compose/resources/prometheus.yml b/lib/docker_compose/resources/prometheus.yml
index 623033be..7690211d 100644
--- a/lib/docker_compose/resources/prometheus.yml
+++ b/lib/docker_compose/resources/prometheus.yml
@@ -5,13 +5,26 @@ scrape_configs:
- job_name: 'zookeeper'
static_configs:
- targets: ['zookeeper:7000']
- - job_name: 'coordinator'
+
+ - job_name: 'grid-java-coordinator'
metrics_path: '/coordinator/prometheus'
scrape_interval: 5s
static_configs:
- targets: ['grid-java-coordinator:8081']
+
- job_name: 'grid'
metrics_path: '/grid/prometheus'
scrape_interval: 5s
static_configs:
- targets: ['grid-java-api:8080']
+
+ - job_name: 'grid-java-observer'
+ metrics_path: '/grid-observer/prometheus'
+ scrape_interval: 5s
+ static_configs:
+ - targets: ['grid-java-observer:8082']
+
+ - job_name: 'clickhouse'
+ scrape_interval: 2s
+ static_configs:
+ - targets: ['clickhouse:9363']
\ No newline at end of file
diff --git a/lib/services/aws/aws-resources/clusterServiceStackTemplate.yaml b/lib/services/aws/aws-resources/clusterServiceStackTemplate.yaml
index 5489753c..a1908a95 100644
--- a/lib/services/aws/aws-resources/clusterServiceStackTemplate.yaml
+++ b/lib/services/aws/aws-resources/clusterServiceStackTemplate.yaml
@@ -327,6 +327,8 @@ Resources:
Value: !Ref ClusterVpcId
- Name: BROWSERUP_CLUSTER_NAME
Value: !Ref ClusterName
+ - Name: BROWSERUP_OBSERVER_VERSION
+ Value: !Ref ServicesImageTag
- Name: EC2_INSTANCE_FAMILY
Value: c5
- Name: BROWSERUP_INFLUX_DB_HOST
@@ -359,6 +361,10 @@ Resources:
Value: 9000
- Name: BROWSERUP_S3_MINIO_PORT_2
Value: 9001
+ - Name: BROWSERUP_LOKI_HOST
+ Value: !Join ['', ['loki.', !Ref PrivateDnsNamespaceName]]
+ - Name: BROWSERUP_LOKI_PORT
+ Value: 3100
- Name: BROWSERUP_S3_MINIO_ACCESS_KEY_ID
Value: !Ref S3MinioAccessKeyID
- Name: BROWSERUP_S3_MINIO_SECRET_ACCESS_KEY
diff --git a/lib/services/docker_client.mjs b/lib/services/docker_client.mjs
index b0da79ab..39348e9e 100644
--- a/lib/services/docker_client.mjs
+++ b/lib/services/docker_client.mjs
@@ -319,14 +319,14 @@ export class DockerClient {
return;
}
- const cmd = `docker stop ${minionsContainersIds.join(' ')};`;
+ const cmd = `docker stop ${minionsContainersIds.join(' ')}`;
try {
await this.execCommand(cmd);
} catch (e) {
throw decoratedError({msg: `Failed to stop minion containers`, error: e});
}
- const cmd2 = `docker rm ${minionsContainersIds.join(' ')};`;
+ const cmd2 = `docker rm ${minionsContainersIds.join(' ')}`;
try {
await this.execCommand(cmd2);
} catch (errors) {
diff --git a/lib/services/local_cluster_pilot.mjs b/lib/services/local_cluster_pilot.mjs
index 6d75a124..20ff105d 100644
--- a/lib/services/local_cluster_pilot.mjs
+++ b/lib/services/local_cluster_pilot.mjs
@@ -16,8 +16,6 @@ export class LocalClusterPilot extends ClusterPilot {
let services = {
"zookeeper": LocalClusterParams.ZOOKEEPER_PORT,
"rabbitmq": LocalClusterParams.RABBITMQ_PORT,
- "influxdb": LocalClusterParams.INFLUXDB_PORT,
- "mysql": LocalClusterParams.BROWSERUP_MYSQL_PORT,
"webconsole": LocalClusterParams.BROWSERUP_WEBCONSOLE_PORT
};
checkLocalPortsAreFree(services);
@@ -39,11 +37,10 @@ export class LocalClusterPilot extends ClusterPilot {
});
ClusterCredentialsRepository.saveCredentials(clusterCredentials);
const composeYmlFile = BrowserUpPaths.dockerComposeYmlPath();
- const backendServices = ["zookeeper", "rabbitmq", "influxdb", "mysql"];
- if (process.env.CHRONOGRAF) backendServices.push("chrono");
+ const backendServices = ["zookeeper", "rabbitmq", "clickhouse"];
const frontendServices = [];
if (!process.env.DEBUG_WEBCONSOLE) frontendServices.push("webconsole");
- const middlewareServices = ["grid-java-coordinator", "grid-java-api"];
+ const middlewareServices = ["grid-java-coordinator", "grid-java-api", "grid-java-observer", "grafana"];
const allServices = backendServices.concat(middlewareServices, frontendServices);
const envs = {
diff --git a/lib/utils/docker.mjs b/lib/utils/docker.mjs
index 3267727a..21622e26 100644
--- a/lib/utils/docker.mjs
+++ b/lib/utils/docker.mjs
@@ -92,7 +92,7 @@ export function copyFilesToDotFolderIfMissing(servicesImageTag) {
servicesImageTag = servicesImageTag || SERVICES_VERSION;
- DcConfigEditor.updateServicesImageTags(dcYmlPath, ["grid-java-api", "grid-java-coordinator", "webconsole"], servicesImageTag);
+ DcConfigEditor.updateServicesImageTags(dcYmlPath, ["grid-java-api", "grid-java-coordinator", "webconsole", "grid-java-observer"], servicesImageTag);
DcConfigEditor.updateEnvForService(
dcYmlPath,
diff --git a/lib/utils/local_env_vars.mjs b/lib/utils/local_env_vars.mjs
index 383b377d..902f0ed9 100644
--- a/lib/utils/local_env_vars.mjs
+++ b/lib/utils/local_env_vars.mjs
@@ -7,8 +7,6 @@ export class LocalEnvVars {
static testEnv() {
return {
- "BROWSERUP_MYSQL_PASSWORD": "just_testing",
- "BROWSERUP_INFLUX_DB_PASSWORD": this.getOrCreateSecret("BROWSERUP_INFLUX_DB_PASSWORD"),
"BROWSERUP_RABBITMQ_DEFAULT_PASS": "just_testing",
"BROWSERUP_API_URL": "http://localhost:3000"
};
diff --git a/state b/state
new file mode 100644
index 00000000..31d2e419
Binary files /dev/null and b/state differ