{
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": {"type": "grafana", "uid": "-- Grafana --"},
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "description": "Operational dashboard for Kafka producer internals as exposed by Micrometer's KafkaClientMetrics binder. Visualizes throughput, request latency, buffer saturation, errors, retries and connection churn — disaggregated per topic class. Requires the appender's bindMeterRegistry() to have wired the Kafka binder, i.e. micrometer-core's KafkaClientMetrics on the classpath. See the appender's Metrics section in the README for setup.",
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": null,
  "links": [
    {
      "asDropdown": false,
      "icon": "external link",
      "includeVars": true,
      "keepTime": true,
      "tags": ["kafka", "logback", "logging"],
      "targetBlank": false,
      "title": "Appender dashboard",
      "tooltip": "Open the logback-kafka-appender dashboard",
      "type": "dashboards"
    }
  ],
  "liveNow": false,
  "panels": [
    {
      "collapsed": false,
      "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0},
      "id": 100,
      "panels": [],
      "title": "Producer health overview",
      "type": "row"
    },
    {
      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
      "description": "Aggregate outgoing throughput to the Kafka cluster, summed across all topic classes. Compare with the appender's events.dispatched rate — a large gap suggests batching is collapsing many small events into fewer broker requests (which is the intended behavior with linger.ms set).",
      "fieldConfig": {
        "defaults": {
          "color": {"mode": "thresholds"},
          "thresholds": {
            "mode": "absolute",
            "steps": [{"color": "green", "value": null}]
          },
          "unit": "Bps"
        }
      },
      "gridPos": {"h": 4, "w": 6, "x": 0, "y": 1},
      "id": 1,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
        "textMode": "auto"
      },
      "pluginVersion": "10.0.0",
      "targets": [
        {
          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
          "editorMode": "code",
          "expr": "sum(kafka_producer_outgoing_byte_rate{application=~\"$application\",topic_class=~\"$topic_class\"})",
          "legendFormat": "outgoing",
          "refId": "A"
        }
      ],
      "title": "Outgoing bytes/s",
      "type": "stat"
    },
    {
      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
      "description": "Records the producer accepted for delivery per second, summed across topic classes. The producer reports a record as 'sent' once it has been buffered or transmitted — final delivery confirmation is in record.send.total (separate counter).",
      "fieldConfig": {
        "defaults": {
          "color": {"mode": "thresholds"},
          "thresholds": {
            "mode": "absolute",
            "steps": [{"color": "green", "value": null}]
          },
          "unit": "reqps"
        }
      },
      "gridPos": {"h": 4, "w": 6, "x": 6, "y": 1},
      "id": 2,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
        "textMode": "auto"
      },
      "pluginVersion": "10.0.0",
      "targets": [
        {
          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
          "editorMode": "code",
          "expr": "sum(kafka_producer_record_send_rate{application=~\"$application\",topic_class=~\"$topic_class\"})",
          "legendFormat": "records/s",
          "refId": "A"
        }
      ],
      "title": "Records sent/s",
      "type": "stat"
    },
    {
      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
      "description": "Aggregate record-error rate. Errors here are individual record rejections by the broker — typically RecordTooLargeException, InvalidTopicException, or SerializationException. These are 'ignored' by the appender's circuit breaker by design (they are deterministic client-side errors, not infrastructure problems), so they will not trip the breaker but they DO end up in the appender's events.fallback{reason=send.error} counter.",
      "fieldConfig": {
        "defaults": {
          "color": {"mode": "thresholds"},
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {"color": "green", "value": null},
              {"color": "yellow", "value": 0.01},
              {"color": "red", "value": 1}
            ]
          },
          "unit": "reqps"
        }
      },
      "gridPos": {"h": 4, "w": 6, "x": 12, "y": 1},
      "id": 3,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
        "textMode": "auto"
      },
      "pluginVersion": "10.0.0",
      "targets": [
        {
          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
          "editorMode": "code",
          "expr": "sum(kafka_producer_record_error_rate{application=~\"$application\",topic_class=~\"$topic_class\"})",
          "legendFormat": "errors/s",
          "refId": "A"
        }
      ],
      "title": "Record errors/s",
      "type": "stat"
    },
    {
      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
      "description": "Average request latency from sending a produce request to the broker until the acknowledgement is received. With a healthy local cluster, single-digit milliseconds. Sustained values above 100 ms suggest broker congestion, network latency, or — when correlated with high record sizes — large-payload effects.",
      "fieldConfig": {
        "defaults": {
          "color": {"mode": "thresholds"},
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {"color": "green", "value": null},
              {"color": "yellow", "value": 50},
              {"color": "red", "value": 200}
            ]
          },
          "unit": "ms"
        }
      },
      "gridPos": {"h": 4, "w": 6, "x": 18, "y": 1},
      "id": 4,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
        "textMode": "auto"
      },
      "pluginVersion": "10.0.0",
      "targets": [
        {
          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
          "editorMode": "code",
          "expr": "avg(kafka_producer_request_latency_avg{application=~\"$application\",topic_class=~\"$topic_class\"})",
          "legendFormat": "avg latency",
          "refId": "A"
        }
      ],
      "title": "Request latency (avg)",
      "type": "stat"
    },
    {
      "collapsed": false,
      "gridPos": {"h": 1, "w": 24, "x": 0, "y": 5},
      "id": 101,
      "panels": [],
      "title": "Throughput",
      "type": "row"
    },
    {
      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
      "description": "Outgoing bytes/s per topic class. AUDIT typically sits low; PERFORMANCE typically dominates. Sustained drops without a corresponding drop in record.send.rate suggest payloads are being compressed more aggressively (often a good thing).",
      "fieldConfig": {
        "defaults": {
          "color": {"mode": "palette-classic"},
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 20,
            "gradientMode": "none",
            "hideFrom": {"legend": false, "tooltip": false, "viz": false},
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {"type": "linear"},
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {"group": "A", "mode": "normal"},
            "thresholdsStyle": {"mode": "off"}
          },
          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
          "unit": "Bps"
        }
      },
      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6},
      "id": 5,
      "options": {
        "legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "right", "showLegend": true},
        "tooltip": {"mode": "multi", "sort": "desc"}
      },
      "targets": [
        {
          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
          "editorMode": "code",
          "expr": "sum by (topic_class) (kafka_producer_outgoing_byte_rate{application=~\"$application\",topic_class=~\"$topic_class\"})",
          "legendFormat": "{{topic_class}}",
          "refId": "A"
        }
      ],
      "title": "Outgoing bytes/s by topic class",
      "type": "timeseries"
    },
    {
      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
      "description": "Average batch size and average record size, side by side. A growing gap (large batch, small record) is healthy: linger.ms is doing its job and the producer is amortizing network costs across many small records. A shrinking gap (batch ≈ record size) means linger.ms is being defeated by request frequency — usually fine for AUDIT/FUNCTIONAL where each record matters.",
      "fieldConfig": {
        "defaults": {
          "color": {"mode": "palette-classic"},
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {"legend": false, "tooltip": false, "viz": false},
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {"type": "linear"},
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {"group": "A", "mode": "none"},
            "thresholdsStyle": {"mode": "off"}
          },
          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
          "unit": "bytes"
        }
      },
      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6},
      "id": 6,
      "options": {
        "legend": {"calcs": ["lastNotNull", "mean"], "displayMode": "table", "placement": "right", "showLegend": true},
        "tooltip": {"mode": "multi", "sort": "desc"}
      },
      "targets": [
        {
          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
          "editorMode": "code",
          "expr": "avg by (topic_class) (kafka_producer_batch_size_avg{application=~\"$application\",topic_class=~\"$topic_class\"})",
          "legendFormat": "batch.size.avg {{topic_class}}",
          "refId": "A"
        },
        {
          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
          "editorMode": "code",
          "expr": "avg by (topic_class) (kafka_producer_record_size_avg{application=~\"$application\",topic_class=~\"$topic_class\"})",
          "legendFormat": "record.size.avg {{topic_class}}",
          "refId": "B"
        }
      ],
      "title": "Batch size vs. record size",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {"h": 1, "w": 24, "x": 0, "y": 14},
      "id": 102,
      "panels": [],
      "title": "Latency",
      "type": "row"
    },
    {
      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
      "description": "Per-class request latency, both average and worst-case. The max line is the more diagnostic one — a spike there typically precedes any visible average degradation. Sustained max > 1 second is a strong signal of broker or network trouble.\n\nNote: this is the producer's view of the network round-trip, NOT the appender's wall-clock send.duration. The two should correlate but the producer's value excludes buffer-wait time.",
      "fieldConfig": {
        "defaults": {
          "color": {"mode": "palette-classic"},
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {"legend": false, "tooltip": false, "viz": false},
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {"type": "linear"},
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {"group": "A", "mode": "none"},
            "thresholdsStyle": {"mode": "off"}
          },
          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
          "unit": "ms"
        }
      },
      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 15},
      "id": 7,
      "options": {
        "legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "right", "showLegend": true},
        "tooltip": {"mode": "multi", "sort": "desc"}
      },
      "targets": [
        {
          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
          "editorMode": "code",
          "expr": "avg by (topic_class) (kafka_producer_request_latency_avg{application=~\"$application\",topic_class=~\"$topic_class\"})",
          "legendFormat": "avg {{topic_class}}",
          "refId": "A"
        },
        {
          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
          "editorMode": "code",
          "expr": "max by (topic_class) (kafka_producer_request_latency_max{application=~\"$application\",topic_class=~\"$topic_class\"})",
          "legendFormat": "max {{topic_class}}",
          "refId": "B"
        }
      ],
      "title": "Request latency",
      "type": "timeseries"
    },
    {
      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
      "description": "Time a record spends in the producer's accumulator queue before being sent. This is largely determined by linger.ms and batch.size — the producer waits up to linger.ms to fill a batch. High queue times are normal when traffic is sparse; very low queue times mean the producer is sending immediately, which is inefficient.\n\nHigh max values without high average indicate occasional slow batches — usually correlated with broker request latency spikes.",
      "fieldConfig": {
        "defaults": {
          "color": {"mode": "palette-classic"},
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {"legend": false, "tooltip": false, "viz": false},
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {"type": "linear"},
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {"group": "A", "mode": "none"},
            "thresholdsStyle": {"mode": "off"}
          },
          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
          "unit": "ms"
        }
      },
      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 15},
      "id": 8,
      "options": {
        "legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "right", "showLegend": true},
        "tooltip": {"mode": "multi", "sort": "desc"}
      },
      "targets": [
        {
          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
          "editorMode": "code",
          "expr": "avg by (topic_class) (kafka_producer_record_queue_time_avg{application=~\"$application\",topic_class=~\"$topic_class\"})",
          "legendFormat": "avg {{topic_class}}",
          "refId": "A"
        },
        {
          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
          "editorMode": "code",
          "expr": "max by (topic_class) (kafka_producer_record_queue_time_max{application=~\"$application\",topic_class=~\"$topic_class\"})",
          "legendFormat": "max {{topic_class}}",
          "refId": "B"
        }
      ],
      "title": "Record queue time",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {"h": 1, "w": 24, "x": 0, "y": 23},
      "id": 103,
      "panels": [],
      "title": "Buffer pool",
      "type": "row"
    },
    {
      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
      "description": "Producer buffer pool saturation. Each producer has a fixed buffer (default 32 MB per producer); when it fills up, send() blocks for up to max.block.ms (the appender sets this to 500 ms by default). Sustained high utilization is the leading indicator of impending caller-thread blocking and, eventually, fallback routing.",
      "fieldConfig": {
        "defaults": {
          "color": {"mode": "thresholds"},
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 20,
            "gradientMode": "scheme",
            "hideFrom": {"legend": false, "tooltip": false, "viz": false},
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {"type": "linear"},
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {"group": "A", "mode": "none"},
            "thresholdsStyle": {"mode": "line"}
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {"color": "green", "value": null},
              {"color": "yellow", "value": 0.5},
              {"color": "red", "value": 0.8}
            ]
          },
          "unit": "percentunit",
          "max": 1,
          "min": 0
        }
      },
      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 24},
      "id": 9,
      "options": {
        "legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "right", "showLegend": true},
        "tooltip": {"mode": "multi", "sort": "desc"}
      },
      "targets": [
        {
          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
          "editorMode": "code",
          "expr": "1 - (kafka_producer_buffer_available_bytes{application=~\"$application\",topic_class=~\"$topic_class\"} / clamp_min(kafka_producer_buffer_total_bytes{application=~\"$application\",topic_class=~\"$topic_class\"}, 1))",
          "legendFormat": "{{topic_class}}",
          "refId": "A"
        }
      ],
      "title": "Buffer saturation",
      "type": "timeseries"
    },
    {
      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
      "description": "Fraction of time the producer's append() was blocked waiting for buffer space. A non-zero value here is the canonical 'producer back-pressure' signal — your application threads are being held in send() for buffer allocation. Sustained values above 1 % suggest the buffer is too small or the broker is too slow.\n\nFor reactive applications (Reactor-Netty event-loop threads), even brief spikes here are concerning: see the appender's 'Reactive applications' section in the README.",
      "fieldConfig": {
        "defaults": {
          "color": {"mode": "thresholds"},
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {"legend": false, "tooltip": false, "viz": false},
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {"type": "linear"},
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {"group": "A", "mode": "none"},
            "thresholdsStyle": {"mode": "line"}
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {"color": "green", "value": null},
              {"color": "yellow", "value": 0.01},
              {"color": "red", "value": 0.05}
            ]
          },
          "unit": "percentunit",
          "max": 1,
          "min": 0
        }
      },
      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 24},
      "id": 10,
      "options": {
        "legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "right", "showLegend": true},
        "tooltip": {"mode": "multi", "sort": "desc"}
      },
      "targets": [
        {
          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
          "editorMode": "code",
          "expr": "avg by (topic_class) (kafka_producer_bufferpool_wait_ratio{application=~\"$application\",topic_class=~\"$topic_class\"})",
          "legendFormat": "{{topic_class}}",
          "refId": "A"
        }
      ],
      "title": "Buffer-wait ratio",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {"h": 1, "w": 24, "x": 0, "y": 32},
      "id": 104,
      "panels": [],
      "title": "Errors & retries",
      "type": "row"
    },
    {
      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
      "description": "Record errors per topic class. Compare with the appender's fallback{reason=send.error} counter — they should match. Cross-correlation with batch size and message size points at root cause (e.g. RecordTooLargeException correlates with large batch sizes that exceed message.max.bytes on the broker).",
      "fieldConfig": {
        "defaults": {
          "color": {"mode": "palette-classic"},
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "none",
            "hideFrom": {"legend": false, "tooltip": false, "viz": false},
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {"type": "linear"},
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {"group": "A", "mode": "none"},
            "thresholdsStyle": {"mode": "off"}
          },
          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
          "unit": "reqps"
        }
      },
      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 33},
      "id": 11,
      "options": {
        "legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "right", "showLegend": true},
        "tooltip": {"mode": "multi", "sort": "desc"}
      },
      "targets": [
        {
          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
          "editorMode": "code",
          "expr": "sum by (topic_class) (kafka_producer_record_error_rate{application=~\"$application\",topic_class=~\"$topic_class\"})",
          "legendFormat": "{{topic_class}}",
          "refId": "A"
        }
      ],
      "title": "Record errors/s by topic class",
      "type": "timeseries"
    },
    {
      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
      "description": "Retry attempts per second per topic class. Retries are transparent to the appender (the breaker only sees the final outcome), but a high retry rate is an early signal that the cluster is degraded — broker leader election in progress, transient network partitions, etc. Compare with batch error rates: many retries with few errors means the producer is recovering, many retries with rising errors means the cluster is not.",
      "fieldConfig": {
        "defaults": {
          "color": {"mode": "palette-classic"},
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {"legend": false, "tooltip": false, "viz": false},
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {"type": "linear"},
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {"group": "A", "mode": "none"},
            "thresholdsStyle": {"mode": "off"}
          },
          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
          "unit": "reqps"
        }
      },
      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 33},
      "id": 12,
      "options": {
        "legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "right", "showLegend": true},
        "tooltip": {"mode": "multi", "sort": "desc"}
      },
      "targets": [
        {
          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
          "editorMode": "code",
          "expr": "sum by (topic_class) (kafka_producer_record_retry_rate{application=~\"$application\",topic_class=~\"$topic_class\"})",
          "legendFormat": "{{topic_class}}",
          "refId": "A"
        }
      ],
      "title": "Record retries/s by topic class",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {"h": 1, "w": 24, "x": 0, "y": 41},
      "id": 105,
      "panels": [],
      "title": "Connections & I/O",
      "type": "row"
    },
    {
      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
      "description": "Active TCP connections from the producer to brokers. A healthy producer maintains a connection per broker for the partitions it writes to — typically a small constant. Spikes indicate connection churn (broker restarts, network instability); sustained drops suggest the producer cannot reach part of the cluster.",
      "fieldConfig": {
        "defaults": {
          "color": {"mode": "palette-classic"},
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {"legend": false, "tooltip": false, "viz": false},
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {"type": "linear"},
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {"group": "A", "mode": "none"},
            "thresholdsStyle": {"mode": "off"}
          },
          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
          "unit": "short"
        }
      },
      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 42},
      "id": 13,
      "options": {
        "legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "right", "showLegend": true},
        "tooltip": {"mode": "multi", "sort": "desc"}
      },
      "targets": [
        {
          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
          "editorMode": "code",
          "expr": "sum by (topic_class) (kafka_producer_connection_count{application=~\"$application\",topic_class=~\"$topic_class\"})",
          "legendFormat": "{{topic_class}}",
          "refId": "A"
        }
      ],
      "title": "Active connections",
      "type": "timeseries"
    },
    {
      "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
      "description": "Connection churn: how often the producer opens new connections vs. closes existing ones. In steady state both should be near zero (connections are kept alive). Spikes correlate with broker restarts, controller failovers, or — at startup — initial metadata bootstrap.",
      "fieldConfig": {
        "defaults": {
          "color": {"mode": "palette-classic"},
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {"legend": false, "tooltip": false, "viz": false},
            "lineInterpolation": "linear",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {"type": "linear"},
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {"group": "A", "mode": "none"},
            "thresholdsStyle": {"mode": "off"}
          },
          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
          "unit": "reqps"
        }
      },
      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 42},
      "id": 14,
      "options": {
        "legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "right", "showLegend": true},
        "tooltip": {"mode": "multi", "sort": "desc"}
      },
      "targets": [
        {
          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
          "editorMode": "code",
          "expr": "sum by (topic_class) (kafka_producer_connection_creation_rate{application=~\"$application\",topic_class=~\"$topic_class\"})",
          "legendFormat": "open/s {{topic_class}}",
          "refId": "A"
        },
        {
          "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
          "editorMode": "code",
          "expr": "sum by (topic_class) (kafka_producer_connection_close_rate{application=~\"$application\",topic_class=~\"$topic_class\"})",
          "legendFormat": "close/s {{topic_class}}",
          "refId": "B"
        }
      ],
      "title": "Connection churn (open vs. close rate)",
      "type": "timeseries"
    }
  ],
  "refresh": "30s",
  "schemaVersion": 38,
  "style": "dark",
  "tags": ["kafka", "logback", "logging", "producer"],
  "templating": {
    "list": [
      {
        "current": {"selected": false, "text": "Prometheus", "value": "Prometheus"},
        "hide": 0,
        "includeAll": false,
        "label": "Datasource",
        "multi": false,
        "name": "DS_PROMETHEUS",
        "options": [],
        "query": "prometheus",
        "queryValue": "",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "type": "datasource"
      },
      {
        "current": {"selected": false, "text": "All", "value": "$__all"},
        "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
        "definition": "label_values(kafka_producer_outgoing_byte_rate, application)",
        "hide": 0,
        "includeAll": true,
        "label": "Application",
        "multi": true,
        "name": "application",
        "options": [],
        "query": {
          "query": "label_values(kafka_producer_outgoing_byte_rate, application)",
          "refId": "PrometheusVariableQueryEditor-VariableQuery"
        },
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "type": "query"
      },
      {
        "current": {"selected": false, "text": "All", "value": "$__all"},
        "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"},
        "definition": "label_values(kafka_producer_outgoing_byte_rate{application=~\"$application\"}, topic_class)",
        "hide": 0,
        "includeAll": true,
        "label": "Topic class",
        "multi": true,
        "name": "topic_class",
        "options": [],
        "query": {
          "query": "label_values(kafka_producer_outgoing_byte_rate{application=~\"$application\"}, topic_class)",
          "refId": "PrometheusVariableQueryEditor-VariableQuery"
        },
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "type": "query"
      }
    ]
  },
  "time": {"from": "now-1h", "to": "now"},
  "timepicker": {},
  "timezone": "",
  "title": "Kafka Producer Internals",
  "uid": "kafka-producer-internals",
  "version": 1,
  "weekStart": ""
}