{
  "kind": "warmup-observer-incidents",
  "exportedAt": "2026-10-10T22:12:40.264Z",
  "ledger": {
    "lo": 0,
    "hi": 16260,
    "n": 16261
  },
  "incidents": [
    {
      "id": 1,
      "type": "node_unreachable",
      "severity": "warning",
      "dedup_key": "node_unreachable",
      "ref": null,
      "block_from": null,
      "block_to": null,
      "title": "本地节点无法连接",
      "opened_at": 1791275344192,
      "resolved_at": 1791275350281,
      "last_seen_at": 1791275344192,
      "occurrences": 1,
      "evidence": {
        "rpc": "LOCAL_RPC",
        "error": "network error: ECONNREFUSED",
        "lastOkAt": null,
        "since": 1791275332789,
        "resolution": "本地节点已恢复连接"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 2,
      "type": "lagging",
      "severity": "warning",
      "dedup_key": "lagging",
      "ref": null,
      "block_from": null,
      "block_to": null,
      "title": "落后参考源超过 10 个区块",
      "opened_at": 1791275382983,
      "resolved_at": 1791275391987,
      "last_seen_at": 1791275382983,
      "occurrences": 1,
      "evidence": {
        "localHead": 366,
        "bestReferenceHead": 771,
        "lag": 405,
        "maxLag": 10,
        "since": 1791275350551,
        "resolution": "已追上参考源"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 3,
      "type": "no_peers",
      "severity": "warning",
      "dedup_key": "no_peers",
      "ref": null,
      "block_from": null,
      "block_to": null,
      "title": "Local node has no peers",
      "opened_at": 1791289917665,
      "resolved_at": 1791289952539,
      "last_seen_at": 1791289930917,
      "occurrences": 2,
      "evidence": {
        "peers": 0,
        "since": 1791289881822,
        "resolution": "peers connected again"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 4,
      "type": "reference_unavailable",
      "severity": "warning",
      "dedup_key": "reference_unavailable:warmup-rpc.levelup.land",
      "ref": "warmup-rpc.levelup.land",
      "block_from": null,
      "block_to": null,
      "title": "Reference warmup-rpc.levelup.land unreachable",
      "opened_at": 1791302593750,
      "resolved_at": 1791302599318,
      "last_seen_at": 1791302593750,
      "occurrences": 1,
      "evidence": {
        "ref": "warmup-rpc.levelup.land",
        "error": "eth_blockNumber: upstream unavailable",
        "lastOkAt": 1791302580244,
        "missingBlocks": [],
        "resolution": "reference available again"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 5,
      "type": "reference_unavailable",
      "severity": "warning",
      "dedup_key": "reference_unavailable:warmup-rpc.levelup.land",
      "ref": "warmup-rpc.levelup.land",
      "block_from": null,
      "block_to": null,
      "title": "Reference warmup-rpc.levelup.land unreachable",
      "opened_at": 1791306578263,
      "resolved_at": 1791309796252,
      "last_seen_at": 1791309789991,
      "occurrences": 616,
      "evidence": {
        "ref": "warmup-rpc.levelup.land",
        "error": "eth_blockNumber: HTTP 530, unexpected reply",
        "lastOkAt": 1791306562241,
        "missingBlocks": [],
        "resolution": "reference available again"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 6,
      "type": "reference_unavailable",
      "severity": "warning",
      "dedup_key": "reference_unavailable:warmup-scan.levelup.land",
      "ref": "warmup-scan.levelup.land",
      "block_from": null,
      "block_to": null,
      "title": "Reference warmup-scan.levelup.land unreachable",
      "opened_at": 1791306584303,
      "resolved_at": 1791309796128,
      "last_seen_at": 1791309789976,
      "occurrences": 615,
      "evidence": {
        "ref": "warmup-scan.levelup.land",
        "error": "eth_blockNumber: HTTP 530, unexpected reply",
        "lastOkAt": 1791306567564,
        "missingBlocks": [],
        "resolution": "reference available again"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 7,
      "type": "no_peers",
      "severity": "warning",
      "dedup_key": "no_peers",
      "ref": null,
      "block_from": null,
      "block_to": null,
      "title": "Local node has no peers",
      "opened_at": 1791306623997,
      "resolved_at": 1791309807507,
      "last_seen_at": 1791309801275,
      "occurrences": 610,
      "evidence": {
        "peers": 0,
        "since": 1791306589313,
        "resolution": "peers connected again"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 8,
      "type": "sync_stalled",
      "severity": "warning",
      "dedup_key": "sync_stalled",
      "ref": null,
      "block_from": null,
      "block_to": null,
      "title": "Head not advancing for 180 s",
      "opened_at": 1791306709678,
      "resolved_at": 1791351268275,
      "last_seen_at": 1791351262954,
      "occurrences": 8093,
      "evidence": {
        "head": 1595,
        "headChangedAt": 1791306527907,
        "stallSeconds": 180,
        "since": 1791306709678,
        "resolution": "head advancing again"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 9,
      "type": "lagging",
      "severity": "warning",
      "dedup_key": "lagging",
      "ref": null,
      "block_from": null,
      "block_to": null,
      "title": "More than 10 blocks behind the references",
      "opened_at": 1791309828147,
      "resolved_at": 1791351312469,
      "last_seen_at": 1791351300186,
      "occurrences": 7500,
      "evidence": {
        "localHead": 2217,
        "bestReferenceHead": 2382,
        "lag": 165,
        "maxLag": 10,
        "since": 1791309796253,
        "resolution": "caught up with the references"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 10,
      "type": "reference_unavailable",
      "severity": "warning",
      "dedup_key": "reference_unavailable:warmup-rpc.levelup.land",
      "ref": "warmup-rpc.levelup.land",
      "block_from": null,
      "block_to": null,
      "title": "Reference warmup-rpc.levelup.land unreachable",
      "opened_at": 1791322685057,
      "resolved_at": 1791322701660,
      "last_seen_at": 1791322696370,
      "occurrences": 3,
      "evidence": {
        "ref": "warmup-rpc.levelup.land",
        "error": "eth_blockNumber: upstream unavailable",
        "lastOkAt": 1791322674467,
        "missingBlocks": [],
        "resolution": "reference available again"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 11,
      "type": "node_unreachable",
      "severity": "warning",
      "dedup_key": "node_unreachable",
      "ref": null,
      "block_from": null,
      "block_to": null,
      "title": "Local node unreachable",
      "opened_at": 1791351251806,
      "resolved_at": 1791351257639,
      "last_seen_at": 1791351251806,
      "occurrences": 1,
      "evidence": {
        "rpc": "LOCAL_RPC",
        "error": "network error: ECONNREFUSED",
        "lastOkAt": 1791351235420,
        "since": 1791351241111,
        "resolution": "local node reachable again"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 12,
      "type": "node_unreachable",
      "severity": "warning",
      "dedup_key": "node_unreachable",
      "ref": null,
      "block_from": null,
      "block_to": null,
      "title": "Local node unreachable",
      "opened_at": 1791352808554,
      "resolved_at": 1791352819898,
      "last_seen_at": 1791352813857,
      "occurrences": 2,
      "evidence": {
        "rpc": "LOCAL_RPC",
        "error": "network error: ECONNREFUSED",
        "lastOkAt": null,
        "since": 1791352797144,
        "resolution": "local node reachable again"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 13,
      "type": "node_unreachable",
      "severity": "warning",
      "dedup_key": "node_unreachable",
      "ref": null,
      "block_from": null,
      "block_to": null,
      "title": "Local node unreachable",
      "opened_at": 1791352863667,
      "resolved_at": 1791352869556,
      "last_seen_at": 1791352863667,
      "occurrences": 1,
      "evidence": {
        "rpc": "LOCAL_RPC",
        "error": "network error: ECONNREFUSED",
        "lastOkAt": 1791352847757,
        "since": 1791352853044,
        "resolution": "local node reachable again"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 14,
      "type": "no_peers",
      "severity": "warning",
      "dedup_key": "no_peers",
      "ref": null,
      "block_from": null,
      "block_to": null,
      "title": "Local node has no peers",
      "opened_at": 1791384806424,
      "resolved_at": 1791384817698,
      "last_seen_at": 1791384812068,
      "occurrences": 2,
      "evidence": {
        "peers": 0,
        "since": 1791384772764,
        "resolution": "peers connected again"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 15,
      "type": "no_peers",
      "severity": "warning",
      "dedup_key": "no_peers",
      "ref": null,
      "block_from": null,
      "block_to": null,
      "title": "Local node has no peers",
      "opened_at": 1791384928169,
      "resolved_at": 1791384933482,
      "last_seen_at": 1791384928169,
      "occurrences": 1,
      "evidence": {
        "peers": 0,
        "since": 1791384895445,
        "resolution": "peers connected again"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 16,
      "type": "sync_stalled",
      "severity": "warning",
      "dedup_key": "sync_stalled",
      "ref": null,
      "block_from": null,
      "block_to": null,
      "title": "Local sync stalled: no new block for 180 s while the references keep producing blocks (a problem with this observer, not the chain)",
      "opened_at": 1791384939474,
      "resolved_at": 1791385113688,
      "last_seen_at": 1791385108391,
      "occurrences": 27,
      "evidence": {
        "scope": "local",
        "head": 3437,
        "headChangedAt": 1791384755894,
        "stallSeconds": 180,
        "bestReferenceHead": 3452,
        "referenceHeadChangedAt": 1791385086401,
        "watchdog": {
          "enabled": true,
          "available": true,
          "requests": 1,
          "lastRequestAt": 1791385058148,
          "gaveUp": false
        },
        "since": 1791384939474,
        "resolution": "head advancing again"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 17,
      "type": "lagging",
      "severity": "warning",
      "dedup_key": "lagging",
      "ref": null,
      "block_from": null,
      "block_to": null,
      "title": "15 blocks behind the references: this observer's own sync is behind, the chain is producing blocks",
      "opened_at": 1791385028202,
      "resolved_at": 1791385113689,
      "last_seen_at": 1791385108391,
      "occurrences": 11,
      "evidence": {
        "scope": "local",
        "localHead": 3437,
        "bestReferenceHead": 3452,
        "lag": 15,
        "maxLag": 10,
        "watchdog": {
          "enabled": true,
          "available": true,
          "requests": 1,
          "lastRequestAt": 1791385058148,
          "gaveUp": false
        },
        "since": 1791384994268,
        "resolution": "caught up with the references"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 18,
      "type": "reference_unavailable",
      "severity": "warning",
      "dedup_key": "reference_unavailable:warmup-rpc.levelup.land",
      "ref": "warmup-rpc.levelup.land",
      "block_from": null,
      "block_to": null,
      "title": "Reference warmup-rpc.levelup.land unreachable",
      "opened_at": 1791447299396,
      "resolved_at": 1791447320796,
      "last_seen_at": 1791447299396,
      "occurrences": 1,
      "evidence": {
        "ref": "warmup-rpc.levelup.land",
        "error": "network error: EAI_AGAIN",
        "lastOkAt": 1791447279932,
        "missingBlocks": [],
        "resolution": "reference available again"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 19,
      "type": "no_peers",
      "severity": "warning",
      "dedup_key": "no_peers",
      "ref": null,
      "block_from": null,
      "block_to": null,
      "title": "Local node has no peers",
      "opened_at": 1791467471692,
      "resolved_at": 1791467488387,
      "last_seen_at": 1791467483084,
      "occurrences": 3,
      "evidence": {
        "peers": 0,
        "since": 1791467439046,
        "resolution": "peers connected again"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 20,
      "type": "node_unreachable",
      "severity": "warning",
      "dedup_key": "node_unreachable",
      "ref": null,
      "block_from": null,
      "block_to": null,
      "title": "Local node unreachable",
      "opened_at": 1791655771339,
      "resolved_at": 1791655782898,
      "last_seen_at": 1791655771339,
      "occurrences": 1,
      "evidence": {
        "rpc": "LOCAL_RPC",
        "error": "network error: ECONNREFUSED",
        "lastOkAt": null,
        "since": 1791655752939,
        "resolution": "local node reachable again"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    },
    {
      "id": 21,
      "type": "no_peers",
      "severity": "warning",
      "dedup_key": "no_peers",
      "ref": null,
      "block_from": null,
      "block_to": null,
      "title": "Local node has no peers",
      "opened_at": 1791655818183,
      "resolved_at": 1791655826767,
      "last_seen_at": 1791655818183,
      "occurrences": 1,
      "evidence": {
        "peers": 0,
        "since": 1791655782899,
        "resolution": "peers connected again"
      },
      "acknowledged": false,
      "acknowledged_at": null,
      "ack_note": null
    }
  ]
}