{
  "snapshot": {
    "generatedFromCommit": "f589ba156a04e1023c1a95b622e04800d8e0668e",
    "incidentFocusUtc": "2026-09-09T05:36:00Z/2026-09-09T06:19:59Z",
    "extendedEvidenceUtc": "2026-09-02T00:00:00Z/2026-09-09T17:38:00Z",
    "cluster": "revlock-prod-b",
    "region": "us-east-1",
    "logGroup": "/ecs/revlock-prod-b",
    "redaction": "Customer, job, task, account, consumer, and message identifiers are aliased or omitted."
  },
  "verdict": {
    "provenPrimaryCause": "Unbounded concurrent live-object amplification in processMessageBatch.",
    "provenIntegrityDefect": "Retry-limit cleanup can ACK/delete without first acquiring valid ownership.",
    "longLivedLeak": "Possible but unproven; paired post-full-GC baselines and heap-dominator diffs were not captured.",
    "plausibleRetainer": "Ten-minute task-local reference cache without an explicit entry or byte cap."
  },
  "statistics": {
    "processorOomEventsExtendedWindow": 27,
    "processorOomEventsRollingSevenDays": 23,
    "retryLimitEntriesRemoved": 60,
    "shareAttributedToTopTwoOrgAliasesPercent": 89,
    "freshTaskHeapBeforeClaimMiB": 214.5,
    "freshTaskHeapAfterReferenceLoadMiB": 215.5,
    "freshTaskTimeToFatalMinutesApprox": 3,
    "containerMemoryMiB": 7384,
    "v8OldSpaceMiB": 6872,
    "processorSevenDayAverageCpuPercent": 5.25,
    "normalObservedBatchSize": 20,
    "autoClaimScanSize": 100,
    "retryLimit": 5,
    "referenceCacheTtlMinutes": 10
  },
  "dailyEvidence": [
    {
      "date2026Utc": "Sep 02",
      "processorMemoryPeak": "Not retained in seven-day query",
      "fatalOomEvents": 4,
      "retryLimitDeletes": "Not observed"
    },
    {
      "date2026Utc": "Sep 03",
      "processorMemoryPeak": "99.49%",
      "fatalOomEvents": 11,
      "retryLimitDeletes": 40
    },
    {
      "date2026Utc": "Sep 04",
      "processorMemoryPeak": "99.62%",
      "fatalOomEvents": 2,
      "retryLimitDeletes": 0
    },
    {
      "date2026Utc": "Sep 05",
      "processorMemoryPeak": "99.19%",
      "fatalOomEvents": 3,
      "retryLimitDeletes": 0
    },
    {
      "date2026Utc": "Sep 06",
      "processorMemoryPeak": "99.58%",
      "fatalOomEvents": 2,
      "retryLimitDeletes": 0
    },
    {
      "date2026Utc": "Sep 07",
      "processorMemoryPeak": "99.47%",
      "fatalOomEvents": 0,
      "retryLimitDeletes": 0
    },
    {
      "date2026Utc": "Sep 08",
      "processorMemoryPeak": "76.44%",
      "fatalOomEvents": 0,
      "retryLimitDeletes": 0
    },
    {
      "date2026Utc": "Sep 09",
      "processorMemoryPeak": "99.00%",
      "fatalOomEvents": 5,
      "retryLimitDeletes": 20
    }
  ],
  "fatalEvents": [
    {
      "event": 1,
      "date2026Utc": "Sep 02",
      "timeUtc": "08:56",
      "orgAlias": "Org A",
      "jobAlias": "Job A3"
    },
    {
      "event": 2,
      "date2026Utc": "Sep 02",
      "timeUtc": "09:06",
      "orgAlias": "Org A",
      "jobAlias": "Job A3"
    },
    {
      "event": 3,
      "date2026Utc": "Sep 02",
      "timeUtc": "09:17",
      "orgAlias": "Org A",
      "jobAlias": "Job A3"
    },
    {
      "event": 4,
      "date2026Utc": "Sep 02",
      "timeUtc": "09:27",
      "orgAlias": "Org C",
      "jobAlias": "Job C1"
    },
    {
      "event": 5,
      "date2026Utc": "Sep 03",
      "timeUtc": "05:27",
      "orgAlias": "Org B",
      "jobAlias": "Job B1"
    },
    {
      "event": 6,
      "date2026Utc": "Sep 03",
      "timeUtc": "05:27",
      "orgAlias": "Org B",
      "jobAlias": "Job B1"
    },
    {
      "event": 7,
      "date2026Utc": "Sep 03",
      "timeUtc": "05:37",
      "orgAlias": "Org B",
      "jobAlias": "Job B1"
    },
    {
      "event": 8,
      "date2026Utc": "Sep 03",
      "timeUtc": "08:53",
      "orgAlias": "Org A",
      "jobAlias": "Job A1"
    },
    {
      "event": 9,
      "date2026Utc": "Sep 03",
      "timeUtc": "08:53",
      "orgAlias": "Org A",
      "jobAlias": "Job A1"
    },
    {
      "event": 10,
      "date2026Utc": "Sep 03",
      "timeUtc": "09:03",
      "orgAlias": "Org D",
      "jobAlias": "Job D1"
    },
    {
      "event": 11,
      "date2026Utc": "Sep 03",
      "timeUtc": "09:04",
      "orgAlias": "Org A",
      "jobAlias": "Job A1"
    },
    {
      "event": 12,
      "date2026Utc": "Sep 03",
      "timeUtc": "09:13",
      "orgAlias": "Org A",
      "jobAlias": "Job A1"
    },
    {
      "event": 13,
      "date2026Utc": "Sep 03",
      "timeUtc": "09:13",
      "orgAlias": "Org A",
      "jobAlias": "Job A1"
    },
    {
      "event": 14,
      "date2026Utc": "Sep 03",
      "timeUtc": "09:23",
      "orgAlias": "Org A",
      "jobAlias": "Job A1"
    },
    {
      "event": 15,
      "date2026Utc": "Sep 03",
      "timeUtc": "09:24",
      "orgAlias": "Org C",
      "jobAlias": "Job C2"
    },
    {
      "event": 16,
      "date2026Utc": "Sep 04",
      "timeUtc": "08:57",
      "orgAlias": "Org A",
      "jobAlias": "Job A4"
    },
    {
      "event": 17,
      "date2026Utc": "Sep 04",
      "timeUtc": "09:07",
      "orgAlias": "Org A",
      "jobAlias": "Job A4"
    },
    {
      "event": 18,
      "date2026Utc": "Sep 05",
      "timeUtc": "05:32",
      "orgAlias": "Org B",
      "jobAlias": "Job B2"
    },
    {
      "event": 19,
      "date2026Utc": "Sep 05",
      "timeUtc": "05:32",
      "orgAlias": "Org B",
      "jobAlias": "Job B2"
    },
    {
      "event": 20,
      "date2026Utc": "Sep 05",
      "timeUtc": "05:32",
      "orgAlias": "Org B",
      "jobAlias": "Job B2"
    },
    {
      "event": 21,
      "date2026Utc": "Sep 06",
      "timeUtc": "05:27",
      "orgAlias": "Org B",
      "jobAlias": "Job B3"
    },
    {
      "event": 22,
      "date2026Utc": "Sep 06",
      "timeUtc": "05:30",
      "orgAlias": "Org B",
      "jobAlias": "Job B3"
    },
    {
      "event": 23,
      "date2026Utc": "Sep 09",
      "timeUtc": "05:37",
      "orgAlias": "Org A",
      "jobAlias": "Job A2"
    },
    {
      "event": 24,
      "date2026Utc": "Sep 09",
      "timeUtc": "05:48",
      "orgAlias": "Org A",
      "jobAlias": "Job A2"
    },
    {
      "event": 25,
      "date2026Utc": "Sep 09",
      "timeUtc": "05:59",
      "orgAlias": "Org A",
      "jobAlias": "Job A2"
    },
    {
      "event": 26,
      "date2026Utc": "Sep 09",
      "timeUtc": "06:08",
      "orgAlias": "Org A",
      "jobAlias": "Job A2"
    },
    {
      "event": 27,
      "date2026Utc": "Sep 09",
      "timeUtc": "06:19",
      "orgAlias": "Org A",
      "jobAlias": "Job A2"
    }
  ],
  "localReproducibility": {
    "concurrencyAmplification": "High",
    "retryDeleteOwnershipRace": "High",
    "longLivedRetention": "Medium until sanitized representative fixture exists",
    "exactProductionHeapCurve": "Not required locally; lower old-space while preserving shape",
    "historicalCustomerImpact": "Requires authoritative production reconciliation",
    "existingTestVerification": {
      "verifiedAt": "2026-09-11",
      "suite": "processSalesOrder.pollForMessagesSingleBatchNo.pure.test.js",
      "result": "28 of 28 tests passed",
      "jestRuntimeSeconds": 0.559,
      "notableExistingAssertion": "Retry-limit cleanup ACKs and deletes at delivery count five"
    }
  },
  "requiredFixtureShape": {
    "modifiedOrders": 20,
    "arrangementsPerOrder": "20-22",
    "include": [
      "representative schedules",
      "organization reference data cloned per order",
      "serialized payload byte-size metadata"
    ],
    "exclude": [
      "real customer IDs",
      "real amounts",
      "credentials",
      "raw production documents"
    ]
  },
  "verificationCriteria": [
    "Active work never exceeds configured concurrency in normal, backlog, or reclaimed paths.",
    "Peak heap stays below 70% and RSS below 80% of the selected budget.",
    "Post-TTL/post-full-GC baseline has no sustained positive slope across ten identical cycles.",
    "No consumer ACKs/deletes a message actively owned by another.",
    "Published equals completed plus pending plus durable DLQ.",
    "Replay is idempotent and produces one terminal outcome."
  ],
  "logsInsightsQueries": [
    "fields @timestamp, @logStream, @message | filter @logStream like /prod-processor-b/ and @message like /JavaScript heap out of memory/ | sort @timestamp asc",
    "filter @logStream like /prod-processor-b/ and @message like /Ignoring and removing/ | stats count(*) by bin(1d)",
    "filter @logStream like /prod-processor-b/ and @message like /Successfully claimed messages|Reading from Stream|heap/ | sort @timestamp asc"
  ],
  "sourceArtifacts": [
    {
      "path": "packages/workers/src/processSalesOrder.js",
      "url": "https://github.com/REVLOCK/revanu/blob/f589ba156a04e1023c1a95b622e04800d8e0668e/packages/workers/src/processSalesOrder.js"
    },
    {
      "path": "packages/workers/src/__tests__/processSalesOrder.pollForMessagesSingleBatchNo.pure.test.js",
      "url": "https://github.com/REVLOCK/revanu/blob/f589ba156a04e1023c1a95b622e04800d8e0668e/packages/workers/src/__tests__/processSalesOrder.pollForMessagesSingleBatchNo.pure.test.js"
    },
    {
      "path": "packages/workers/AGENTS.md",
      "url": "https://github.com/REVLOCK/revanu/blob/f589ba156a04e1023c1a95b622e04800d8e0668e/packages/workers/AGENTS.md"
    },
    {
      "path": "packages/svcs/src/ProfileServices.js",
      "url": "https://github.com/REVLOCK/revanu/blob/f589ba156a04e1023c1a95b622e04800d8e0668e/packages/svcs/src/ProfileServices.js"
    },
    {
      "path": "packages/svcs/src/__tests__/ProfileServices.createHeapSnapshotWritesUnderTmpLocal.pure.test.js",
      "url": "https://github.com/REVLOCK/revanu/blob/f589ba156a04e1023c1a95b622e04800d8e0668e/packages/svcs/src/__tests__/ProfileServices.createHeapSnapshotWritesUnderTmpLocal.pure.test.js"
    },
    {
      "path": "packages/workers/src/monitor.js",
      "url": "https://github.com/REVLOCK/revanu/blob/f589ba156a04e1023c1a95b622e04800d8e0668e/packages/workers/src/monitor.js"
    },
    {
      "path": "container/cfg/processorEcsTask.json",
      "url": "https://github.com/REVLOCK/revanu/blob/f589ba156a04e1023c1a95b622e04800d8e0668e/container/cfg/processorEcsTask.json"
    },
    {
      "path": "container/processor/runner.sh",
      "url": "https://github.com/REVLOCK/revanu/blob/f589ba156a04e1023c1a95b622e04800d8e0668e/container/processor/runner.sh"
    },
    {
      "path": "packages/cfg/etc/jobs.json",
      "url": "https://github.com/REVLOCK/revanu/blob/f589ba156a04e1023c1a95b622e04800d8e0668e/packages/cfg/etc/jobs.json"
    },
    {
      "path": "packages/utils/src/MetricTracker.js",
      "url": "https://github.com/REVLOCK/revanu/blob/f589ba156a04e1023c1a95b622e04800d8e0668e/packages/utils/src/MetricTracker.js"
    },
    {
      "path": "packages/utils/src/AsyncContext.js",
      "url": "https://github.com/REVLOCK/revanu/blob/f589ba156a04e1023c1a95b622e04800d8e0668e/packages/utils/src/AsyncContext.js"
    },
    {
      "path": "docs/workflows/sync-pipeline/orders.md",
      "url": "https://github.com/REVLOCK/revanu/blob/f589ba156a04e1023c1a95b622e04800d8e0668e/docs/workflows/sync-pipeline/orders.md"
    },
    {
      "path": "docs/architecture/redis-internals.md",
      "url": "https://github.com/REVLOCK/revanu/blob/f589ba156a04e1023c1a95b622e04800d8e0668e/docs/architecture/redis-internals.md"
    },
    {
      "path": ".agents/skills/cloudwatch-logs/SKILL.md",
      "url": "https://github.com/REVLOCK/revanu/blob/f589ba156a04e1023c1a95b622e04800d8e0668e/.agents/skills/cloudwatch-logs/SKILL.md"
    }
  ],
  "awsArtifacts": [
    "https://us-east-1.console.aws.amazon.com/ecs/v2/clusters/revlock-prod-b/clusterMetrics?region=us-east-1",
    "https://us-east-1.console.aws.amazon.com/ecs/v2/clusters/revlock-prod-b/services?region=us-east-1",
    "https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#logsV2:logs-insights",
    "https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#alarmsV2:"
  ],
  "knownUnknowns": [
    "Authoritative Redis state and counters for affected jobs.",
    "Whether every removed payload exists and is safely replayable.",
    "Downstream document and warehouse completeness.",
    "Heap dominators after work, cache expiry, idle, and full GC.",
    "Representative payload byte-size and graph-sharing distribution."
  ]
}