{"page":{"id":"01KS5BHY7AHH8P8EFW4ZENDP28","name":"Langfuse","url":"https://status.langfuse.com/","updated_at":"2026-09-30T15:30:23Z"},"incidents":[{"id":"01M3RS8E0MFN2SWBTNES3N7EBE","name":"Degraded ingestion for 20 projects","status":"resolved","created_at":"2026-09-30T09:10:17Z","updated_at":"2026-09-30T15:30:23Z","resolved_at":"2026-09-30T15:30:23Z","impact":"major","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01M3SF0CX1CF8CCMBDEFKCRG85","body":"All projects ingesting as normal","created_at":"2026-09-30T15:30:23Z","display_at":"2026-09-30T15:30:23Z","incident_id":"01M3RS8E0MFN2SWBTNES3N7EBE","status":"resolved","updated_at":"2026-09-30T15:30:23Z"},{"id":"01M3RV4PSHSQ13N7WPWC3BSTTJ","body":"Offending workload was isolated, performance should be back to normal for other projects","created_at":"2026-09-30T09:43:12Z","display_at":"2026-09-30T09:43:12Z","incident_id":"01M3RS8E0MFN2SWBTNES3N7EBE","status":"monitoring","updated_at":"2026-09-30T09:43:12Z"},{"id":"01M3RS8E0MTDNJJ677GM50CRVV","body":"S3 backpressure leads to S3 \"please slow down\" errors across projects with cm* prefix","created_at":"2026-09-30T09:10:17Z","display_at":"2026-09-30T09:10:17Z","incident_id":"01M3RS8E0MFN2SWBTNES3N7EBE","status":"identified","updated_at":"2026-09-30T09:10:17Z"}]},{"id":"01M377QR1YYCPPY6PHR89DFGSC","name":"Elevated error rates in In App Agent","status":"resolved","created_at":"2026-09-23T13:36:59Z","updated_at":"2026-09-23T13:43:42Z","resolved_at":"2026-09-23T13:43:42Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01M37840NDD0Y9EXD6M1JSV9BZ","body":"The issue has been resolved.","created_at":"2026-09-23T13:43:42Z","display_at":"2026-09-23T13:43:42Z","incident_id":"01M377QR1YYCPPY6PHR89DFGSC","status":"resolved","updated_at":"2026-09-23T13:43:42Z"},{"id":"01M377QR1YMS6EVKF1KDG706GX","body":"We're observing elevated error rates across our In-App agent across multiple regions. Our team is investigating.","created_at":"2026-09-23T13:36:59Z","display_at":"2026-09-23T13:36:59Z","incident_id":"01M377QR1YYCPPY6PHR89DFGSC","status":"investigating","updated_at":"2026-09-23T13:36:59Z"}]},{"id":"01M2NY0R9X77QMKQ6T4S6CSA3E","name":"OTEL ingestion queue delay over 10 minutes in US","status":"resolved","created_at":"2026-09-16T20:20:04Z","updated_at":"2026-09-16T21:31:38Z","resolved_at":"2026-09-16T21:31:38Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01M2P23SQYR5DKMEXADFNJJ9KH","body":"Issue is resolved and OTEL ingestion pipeline is back to regular ingestion speed.","created_at":"2026-09-16T21:31:38Z","display_at":"2026-09-16T21:31:38Z","incident_id":"01M2NY0R9X77QMKQ6T4S6CSA3E","status":"resolved","updated_at":"2026-09-16T21:31:38Z"},{"id":"01M2NY0R9XNN45ZGN6SHX6MTE5","body":"We're investigating degraded performance in the ingestion flow","created_at":"2026-09-16T20:20:04Z","display_at":"2026-09-16T20:20:04Z","incident_id":"01M2NY0R9X77QMKQ6T4S6CSA3E","status":"investigating","updated_at":"2026-09-16T20:20:04Z"}]},{"id":"01M2EX7FAKVNNVRJ7M6GZK665P","name":"Delayed Ingestion for non v4 SDKs","status":"resolved","created_at":"2026-09-14T02:51:34Z","updated_at":"2026-09-14T06:10:22Z","resolved_at":"2026-09-14T06:10:22Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01M2F8KFKW5VQDHF5QAGP18P6X","body":"The incident has been resolved.","created_at":"2026-09-14T06:10:22Z","display_at":"2026-09-14T06:10:22Z","incident_id":"01M2EX7FAKVNNVRJ7M6GZK665P","status":"resolved","updated_at":"2026-09-14T06:10:22Z"},{"id":"01M2F5DBRYYTMT9WHD6MP1B3AD","body":"We're making consistent progress in our backlog and are currently seeing delays of about 60 minutes.","created_at":"2026-09-14T05:14:36Z","display_at":"2026-09-14T05:14:36Z","incident_id":"01M2EX7FAKVNNVRJ7M6GZK665P","status":"monitoring","updated_at":"2026-09-14T05:14:36Z"},{"id":"01M2F0PCCMQF1MA8JCYKGR35FR","body":"We considerably increased throughput of the batch job to write data into v4 data model. We are currently catching up processing data.","created_at":"2026-09-14T03:52:08Z","display_at":"2026-09-14T03:52:08Z","incident_id":"01M2EX7FAKVNNVRJ7M6GZK665P","status":"investigating","updated_at":"2026-09-14T03:52:08Z"},{"id":"01M2EX7FAK8KVCZZGSB46WV64C","body":"We currently experience an ingestion delay of old SDKs by 3 hours. This is due to performance issues in our batch job which moves data into the v4 data model (\u003chttps://langfuse.com/changelog/2026-08-17-langfuse-v4\u003e).\n\n \n\nWe currently work on improving the batch job. All v4 compatible SDKs are not impacted by this. Upgrading SDKs is recommended.","created_at":"2026-09-14T02:51:34Z","display_at":"2026-09-14T02:51:34Z","incident_id":"01M2EX7FAKVNNVRJ7M6GZK665P","status":"investigating","updated_at":"2026-09-14T02:51:34Z"}]},{"id":"01M210WCHAJ4WKZWDBM4E9XYF5","name":"OTEL Ingestion Delay exceeds 10 minutes","status":"resolved","created_at":"2026-09-08T17:26:03Z","updated_at":"2026-09-08T19:00:55Z","resolved_at":"2026-09-08T19:00:55Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01M216A3BXTXGJDE2BET38TRQM","body":"OTEL ingestion latency spike is resolved","created_at":"2026-09-08T19:00:55Z","display_at":"2026-09-08T19:00:55Z","incident_id":"01M210WCHAJ4WKZWDBM4E9XYF5","status":"resolved","updated_at":"2026-09-08T19:00:55Z"},{"id":"01M210WCHACAXQCF2TATV3T250","body":"We're investigating degraded performance in the ingestion flow","created_at":"2026-09-08T17:26:03Z","display_at":"2026-09-08T17:26:03Z","incident_id":"01M210WCHAJ4WKZWDBM4E9XYF5","status":"investigating","updated_at":"2026-09-08T17:26:03Z"}]},{"id":"01M1HDPTQA3FVQTG8FZWA7DQWG","name":"Degraded OTEL ingestion in US region","status":"resolved","created_at":"2026-09-02T16:02:22Z","updated_at":"2026-09-02T16:56:09Z","resolved_at":"2026-09-02T16:56:09Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01M1HGSAKR93VB14VHZ31D9KDM","body":"Completely back to normal","created_at":"2026-09-02T16:56:09Z","display_at":"2026-09-02T16:56:09Z","incident_id":"01M1HDPTQA3FVQTG8FZWA7DQWG","status":"resolved","updated_at":"2026-09-02T16:56:09Z"},{"id":"01M1HGD2X2YP13V1NG4JW7WXC8","body":"Backlog is almost completely gone. Ingestion delay is \u003c 1 min.","created_at":"2026-09-02T16:49:28Z","display_at":"2026-09-02T16:49:28Z","incident_id":"01M1HDPTQA3FVQTG8FZWA7DQWG","status":"monitoring","updated_at":"2026-09-02T16:49:28Z"},{"id":"01M1HEZGW1JFZBNSDNS4Q4XHGE","body":"A deployment rollout coincided with a traffic spike, resulting in an unusually large backlog. Adjusting our autoscaling to absorb the backlog faster.","created_at":"2026-09-02T16:24:35Z","display_at":"2026-09-02T16:24:35Z","incident_id":"01M1HDPTQA3FVQTG8FZWA7DQWG","status":"identified","updated_at":"2026-09-02T16:24:35Z"},{"id":"01M1HDPTQAKPFXXZB1MB1V2NQF","body":"We are observing OTEL ingestion backlog in US region","created_at":"2026-09-02T16:02:22Z","display_at":"2026-09-02T16:02:22Z","incident_id":"01M1HDPTQA3FVQTG8FZWA7DQWG","status":"investigating","updated_at":"2026-09-02T16:02:22Z"}]},{"id":"01M0WRAE2W2JVQWVHE9K03SS8Z","name":"Performance degradation on ingestion path","status":"resolved","created_at":"2026-08-25T15:23:47Z","updated_at":"2026-08-25T16:28:41Z","resolved_at":"2026-08-25T16:28:41Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01M0WW1987QQDRX8PYEFPNTKCJ","body":"Issue resolved. Latency is back to normal levels.","created_at":"2026-08-25T16:28:41Z","display_at":"2026-08-25T16:28:41Z","incident_id":"01M0WRAE2W2JVQWVHE9K03SS8Z","status":"resolved","updated_at":"2026-08-25T16:28:41Z"},{"id":"01M0WRAE2WZXR7SWY2C5T6A6FK","body":"Ingestion delay of up to multiple minutes on the ingestion path. We have scaled up and are investigating the root cause.","created_at":"2026-08-25T15:23:47Z","display_at":"2026-08-25T15:23:47Z","incident_id":"01M0WRAE2W2JVQWVHE9K03SS8Z","status":"monitoring","updated_at":"2026-08-25T15:23:47Z"}]},{"id":"01M0W6R42TJZ789CC0QNQG971R","name":"Latency degradation on legacy ingestion queue","status":"resolved","created_at":"2026-08-25T10:16:41Z","updated_at":"2026-08-25T13:42:56Z","resolved_at":"2026-08-25T13:42:56Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01M0WJHRYBTDJGP6PQVRQR019G","body":"Issue resolved since 12:30. Legacy ingestion pipeline is fully operational. ","created_at":"2026-08-25T13:42:56Z","display_at":"2026-08-25T13:42:56Z","incident_id":"01M0W6R42TJZ789CC0QNQG971R","status":"resolved","updated_at":"2026-08-25T13:42:56Z"},{"id":"01M0W6R42TN2XT35JSHN37GPTZ","body":"The legacy ingestion queue is experiencing latency degradation up to 5 minutes. We have scaled up our infrastructure in response and are monitoring the situation.\n\nWe recommend to switch to the new OTEL ingestion queue.","created_at":"2026-08-25T10:16:41Z","display_at":"2026-08-25T10:16:41Z","incident_id":"01M0W6R42TJZ789CC0QNQG971R","status":"monitoring","updated_at":"2026-08-25T10:16:41Z"}]},{"id":"01M0F3AAW0WVSYJNF6A2JDW5TM","name":"Evaluations are delayed in EU","status":"resolved","created_at":"2026-08-20T08:06:36Z","updated_at":"2026-08-20T09:55:50Z","resolved_at":"2026-08-20T09:55:50Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01M0F9JBJZ23QHHF3M85NSJP7H","body":"All is back to normal","created_at":"2026-08-20T09:55:50Z","display_at":"2026-08-20T09:55:50Z","incident_id":"01M0F3AAW0WVSYJNF6A2JDW5TM","status":"resolved","updated_at":"2026-08-20T09:55:50Z"},{"id":"01M0F8PBK118XWVRXGKWNQSH37","body":"After concurrency adjustments execution delay is back to normal. Monitoring.","created_at":"2026-08-20T09:40:33Z","display_at":"2026-08-20T09:40:33Z","incident_id":"01M0F3AAW0WVSYJNF6A2JDW5TM","status":"monitoring","updated_at":"2026-08-20T09:40:33Z"},{"id":"01M0F3AAW01HCK8V626ED8ZNXZ","body":"Evaluations are taking longer to execute in EU","created_at":"2026-08-20T08:06:36Z","display_at":"2026-08-20T08:06:36Z","incident_id":"01M0F3AAW0WVSYJNF6A2JDW5TM","status":"investigating","updated_at":"2026-08-20T08:06:36Z"}]},{"id":"01M0D0CCF6N91A79YYMFBGB9WJ","name":"Longer v3 -\u003e v4 propagation delay","status":"resolved","created_at":"2026-08-19T12:36:48Z","updated_at":"2026-08-19T15:10:42Z","resolved_at":"2026-08-19T15:10:42Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01M0D965ZEYN4QTBSF9VEFEZX4","body":"Back to normal propagation delay","created_at":"2026-08-19T15:10:42Z","display_at":"2026-08-19T15:10:42Z","incident_id":"01M0D0CCF6N91A79YYMFBGB9WJ","status":"resolved","updated_at":"2026-08-19T15:10:42Z"},{"id":"01M0D2TD6GF8AX0A2TF2D56B3J","body":"Latest major SDK versions are not impacted. Find upgrade docs [here](https://langfuse.com/docs/observability/sdk/upgrade-path).","created_at":"2026-08-19T13:19:25Z","display_at":"2026-08-19T13:19:25Z","incident_id":"01M0D0CCF6N91A79YYMFBGB9WJ","status":"monitoring","updated_at":"2026-08-19T14:12:42Z"},{"id":"01M0D0CCF6TN370VDK0F1334WY","body":"We have scaled our database infrastructure to match the throughput demand","created_at":"2026-08-19T12:36:48Z","display_at":"2026-08-19T10:30:00Z","incident_id":"01M0D0CCF6N91A79YYMFBGB9WJ","status":"monitoring","updated_at":"2026-08-19T12:37:03Z"}]},{"id":"01M0AC8PM24HN9TVHRWR47G2R2","name":"Degraded LLMaJ execution in EU","status":"resolved","created_at":"2026-08-18T12:06:47Z","updated_at":"2026-08-18T13:25:37Z","resolved_at":"2026-08-18T13:25:37Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01M0AGS1YZ17VSP0S4EPZTEP8E","body":"Steady throughput for the last 30 minutes. Delay \u003c 5min.","created_at":"2026-08-18T13:25:37Z","display_at":"2026-08-18T13:25:37Z","incident_id":"01M0AC8PM24HN9TVHRWR47G2R2","status":"resolved","updated_at":"2026-08-18T13:25:37Z"},{"id":"01M0AEKK5R9Y76R70E6TEQ4VBQ","body":"our autoscaling responded slower than expected, but responded nonetheless. the backlog is draining","created_at":"2026-08-18T12:47:41Z","display_at":"2026-08-18T12:47:41Z","incident_id":"01M0AC8PM24HN9TVHRWR47G2R2","status":"monitoring","updated_at":"2026-08-18T12:47:41Z"},{"id":"01M0ADEK953AT42N5FWAMQTYV1","body":"evaluation throughput was overwhelmed by a slow tenant/model workload, with lock-expiry retries multiplying work; the concurrent worker rollout prolonged the incident.","created_at":"2026-08-18T12:27:29Z","display_at":"2026-08-18T12:27:29Z","incident_id":"01M0AC8PM24HN9TVHRWR47G2R2","status":"identified","updated_at":"2026-08-18T12:27:29Z"},{"id":"01M0AC8PM22B1XPDDYYT9ADAFV","body":"Eval execution in EU region is taking longer than usual","created_at":"2026-08-18T12:06:47Z","display_at":"2026-08-18T12:06:47Z","incident_id":"01M0AC8PM24HN9TVHRWR47G2R2","status":"investigating","updated_at":"2026-08-18T12:06:47Z"}]},{"id":"01M037G9MDS6PM7JNHCWZBFK89","name":"prod-us: Delayed ingestion for prod-us","status":"resolved","created_at":"2026-08-15T17:28:52Z","updated_at":"2026-08-15T18:05:17Z","resolved_at":"2026-08-15T18:05:17Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01M039JZDRHTVDBTQH9VP9CYCE","body":"We incurred a temporary delay on the legacy ingestion path for v4. This delay has been resolved.\n\nOTEL ingestion has not been impacted.","created_at":"2026-08-15T18:05:17Z","display_at":"2026-08-15T18:05:17Z","incident_id":"01M037G9MDS6PM7JNHCWZBFK89","status":"resolved","updated_at":"2026-08-15T18:05:17Z"},{"id":"01M037G9MD3EJ2AV9RMHRQZW24","body":"Currently the ingestion for the events table is delayed in the US environment. This happens only for data from non-latest major SDK versions.\n\nWe are investigating this and have started a mitigation. The data should be up to date soon.","created_at":"2026-08-15T17:28:52Z","display_at":"2026-08-15T17:28:52Z","incident_id":"01M037G9MDS6PM7JNHCWZBFK89","status":"investigating","updated_at":"2026-08-15T17:53:15Z"}]},{"id":"01KZTX9SG4T9GAZMQ2KGWWD649","name":"Latency increase eval execution up to 40 minutes is resolved","status":"resolved","created_at":"2026-08-12T11:56:38Z","updated_at":"2026-08-12T13:37:15Z","resolved_at":"2026-08-12T13:37:15Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01KZV320W3KSNMTE0QZQKY1C0E","body":"The issue is resolved now","created_at":"2026-08-12T13:37:15Z","display_at":"2026-08-12T13:37:15Z","incident_id":"01KZTX9SG4T9GAZMQ2KGWWD649","status":"resolved","updated_at":"2026-08-12T13:37:15Z"},{"id":"01KZTZQX6JW1HKK0R8T1ZDVZ73","body":"Previous latency increase of the eval execution is resolved. Slight delays may still occur.","created_at":"2026-08-12T12:39:18Z","display_at":"2026-08-12T12:39:18Z","incident_id":"01KZTX9SG4T9GAZMQ2KGWWD649","status":"monitoring","updated_at":"2026-08-12T12:39:18Z"},{"id":"01KZTX9SG4SVE22V669RPWBT8K","body":"We are seeing an increase in the eval execution. We have scaled up our infrastructure and are monitoring.","created_at":"2026-08-12T11:56:38Z","display_at":"2026-08-12T11:56:38Z","incident_id":"01KZTX9SG4T9GAZMQ2KGWWD649","status":"monitoring","updated_at":"2026-08-12T11:56:38Z"}]},{"id":"01KZ4DZNVKPJ4Z40WVCGJAMPT4","name":"[US] Ingestion Delays","status":"resolved","created_at":"2026-08-03T18:25:40Z","updated_at":"2026-08-03T19:03:40Z","resolved_at":"2026-08-03T19:03:40Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01KZ4G58330P700D3FS9V485D2","body":"This issue is resolved and we're back to regular processing times.","created_at":"2026-08-03T19:03:40Z","display_at":"2026-08-03T19:03:40Z","incident_id":"01KZ4DZNVKPJ4Z40WVCGJAMPT4","status":"resolved","updated_at":"2026-08-03T19:03:40Z"},{"id":"01KZ4FRZV9GZ0674T9WGJHA7YR","body":"We see an improvement and the current delay is below 5min. We continue to investigate and scale.","created_at":"2026-08-03T18:56:58Z","display_at":"2026-08-03T18:56:58Z","incident_id":"01KZ4DZNVKPJ4Z40WVCGJAMPT4","status":"investigating","updated_at":"2026-08-03T18:56:58Z"},{"id":"01KZ4DZNVK7HC9A8VRMPXX50WF","body":"We're currently observing about 10min ingestion delays. Our team is investigating.","created_at":"2026-08-03T18:25:40Z","display_at":"2026-08-03T18:25:40Z","incident_id":"01KZ4DZNVKPJ4Z40WVCGJAMPT4","status":"investigating","updated_at":"2026-08-03T18:25:40Z"}]},{"id":"01KXH0QCAJ354J5ETHE76SNG49","name":"502 BAD Gateway errors on UI","status":"resolved","created_at":"2026-07-14T19:12:41Z","updated_at":"2026-07-14T20:19:30Z","resolved_at":"2026-07-14T20:19:30Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01KXH4HQHRJ0G4RGTYPRC1W3FW","body":"We resolved the performance bottleneck. All APIs are responding as expected as of now.","created_at":"2026-07-14T20:19:30Z","display_at":"2026-07-14T20:19:30Z","incident_id":"01KXH0QCAJ354J5ETHE76SNG49","status":"resolved","updated_at":"2026-07-14T20:19:30Z"},{"id":"01KXH0QCAJ55DQDH42YBBSRFS2","body":"We experience occasional API errors with 502 Bad Gateway. The team is investigating the root cause currently.","created_at":"2026-07-14T19:12:41Z","display_at":"2026-07-14T19:12:41Z","incident_id":"01KXH0QCAJ354J5ETHE76SNG49","status":"investigating","updated_at":"2026-07-14T19:12:41Z"}]},{"id":"01KWGWFYKT9X16PY7090RSYWYE","name":"[EU] Elevated latencies and error rates","status":"resolved","created_at":"2026-07-02T07:43:01Z","updated_at":"2026-07-02T08:03:36Z","resolved_at":"2026-07-02T08:03:36Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01KWGXNMR21M3YN2Y3XVQGV9E0","body":"The issue has been resolved.","created_at":"2026-07-02T08:03:36Z","display_at":"2026-07-02T08:03:36Z","incident_id":"01KWGWFYKT9X16PY7090RSYWYE","status":"resolved","updated_at":"2026-07-02T08:03:36Z"},{"id":"01KWGWFYKTKFVTDDH7AXGM0E56","body":"We're observing elevated latencies and error rates across read APIs and some frontend routes. Ingestion is not affected.","created_at":"2026-07-02T07:43:01Z","display_at":"2026-07-02T07:43:01Z","incident_id":"01KWGWFYKT9X16PY7090RSYWYE","status":"investigating","updated_at":"2026-07-02T07:43:01Z"}]},{"id":"01KW1MC79474ZC9EJ6NZAQ7T0F","name":"[EU] Elevated ingestion times","status":"resolved","created_at":"2026-06-26T09:32:34Z","updated_at":"2026-06-26T09:51:12Z","resolved_at":"2026-06-26T09:51:12Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01KW1NEANFZK250APB2ZZ6BZPF","body":"The incident has been resolved.","created_at":"2026-06-26T09:51:12Z","display_at":"2026-06-26T09:51:12Z","incident_id":"01KW1MC79474ZC9EJ6NZAQ7T0F","status":"resolved","updated_at":"2026-06-26T09:51:12Z"},{"id":"01KW1N0YZQ0GF3K18SDRGZTAV3","body":"We found a patch and currently work through the backlog. Things should recover within the next 10min.","created_at":"2026-06-26T09:43:54Z","display_at":"2026-06-26T09:43:54Z","incident_id":"01KW1MC79474ZC9EJ6NZAQ7T0F","status":"monitoring","updated_at":"2026-06-26T09:43:54Z"},{"id":"01KW1MC794KC93Y0K9Z919NPD0","body":"We're currently experiencing elevated ingestion times in the EU environment. Some events are processed with a delay of about 15 minutes. Our team is investigating.","created_at":"2026-06-26T09:32:34Z","display_at":"2026-06-26T09:32:34Z","incident_id":"01KW1MC79474ZC9EJ6NZAQ7T0F","status":"investigating","updated_at":"2026-06-26T09:32:34Z"}]},{"id":"01KV8R27KJ44QJGY5R958J3Q0J","name":"Elevated API and tracing latency in the US region","status":"resolved","created_at":"2026-06-16T17:36:00Z","updated_at":"2026-06-16T18:12:46Z","resolved_at":"2026-06-16T18:12:46Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01KV8T5HR2H9C7KV7NHKMDM9AX","body":"All latencies are back to normal","created_at":"2026-06-16T18:12:46Z","display_at":"2026-06-16T18:12:46Z","incident_id":"01KV8R27KJ44QJGY5R958J3Q0J","status":"resolved","updated_at":"2026-06-16T18:12:46Z"},{"id":"01KV8R27KJXTSP800AGSRZB98D","body":"A hiccup during a planned database maintenance. Identified and resolved. Presently monitoring.","created_at":"2026-06-16T17:36:00Z","display_at":"2026-06-16T17:36:00Z","incident_id":"01KV8R27KJ44QJGY5R958J3Q0J","status":"monitoring","updated_at":"2026-06-16T17:36:00Z"}]},{"id":"01KTS015KHNRNVR87Y5Y5BGHQ6","name":"Ingestion delay","status":"resolved","created_at":"2026-06-10T14:47:23Z","updated_at":"2026-06-10T15:16:21Z","resolved_at":"2026-06-10T15:16:21Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01KTS1P6NWV2P2WWAPE4YH0QRY","body":"We scaled our ingestion and are processing data in time.","created_at":"2026-06-10T15:16:21Z","display_at":"2026-06-10T15:16:21Z","incident_id":"01KTS015KHNRNVR87Y5Y5BGHQ6","status":"resolved","updated_at":"2026-06-10T15:16:21Z"},{"id":"01KTS015KH7GM8WT765VT0ZEKR","body":"We currently have a bottleneck in our ingestion pipeline which causes a delay of ingested data of 12 minutes. We currently work to scale the pipeline.","created_at":"2026-06-10T14:47:23Z","display_at":"2026-06-10T14:47:23Z","incident_id":"01KTS015KHNRNVR87Y5Y5BGHQ6","status":"investigating","updated_at":"2026-06-10T14:47:23Z"}]},{"id":"01KTMG9RPS9AV1T412GFAN7QWD","name":"Database upgrdae","status":"resolved","created_at":"2026-06-08T20:55:30Z","updated_at":"2026-06-08T20:55:44Z","resolved_at":"2026-06-08T20:55:44Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01KTMGA6EGYXWMYBF7JZ3SHSE7","body":"Upgrade successfully resolved.\n","created_at":"2026-06-08T20:55:44Z","display_at":"2026-06-08T20:55:44Z","incident_id":"01KTMG9RPS9AV1T412GFAN7QWD","status":"resolved","updated_at":"2026-06-08T20:55:44Z"},{"id":"01KTMG9RPS814TH6YQ4KE9X7EQ","body":"We just executed a database upgrade which caused a downtime of 5 minutes.","created_at":"2026-06-08T20:55:30Z","display_at":"2026-06-08T20:55:30Z","incident_id":"01KTMG9RPS9AV1T412GFAN7QWD","status":"investigating","updated_at":"2026-06-08T20:55:30Z"}]},{"id":"01KTM9HWN7GW0M4Z4E1GXXYXE4","name":"Our media API returns 500 errors","status":"resolved","created_at":"2026-06-08T18:57:36Z","updated_at":"2026-06-10T14:45:43Z","resolved_at":"2026-06-08T19:55:00Z","impact":"major","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01KTRZY3T0JY77V0XR01T24VXM","body":"We scaled our infrastructure and are processing media uploads without errors.","created_at":"2026-06-10T14:45:43Z","display_at":"2026-06-08T19:55:00Z","incident_id":"01KTM9HWN7GW0M4Z4E1GXXYXE4","status":"resolved","updated_at":"2026-06-10T14:46:20Z"},{"id":"01KTMCT3ASDC0KHD2D535MW0WT","body":"We scaled our underlying infrastructure and are processing media uploads without errors.","created_at":"2026-06-08T19:54:31Z","display_at":"2026-06-08T19:54:31Z","incident_id":"01KTM9HWN7GW0M4Z4E1GXXYXE4","status":"investigating","updated_at":"2026-06-08T19:54:31Z"},{"id":"01KTM9HWN7KG00C17NP0FBZ0PM","body":"We observe that our api/public/media endpoint currently returns 500 status errors as our underlying database has performance issues.","created_at":"2026-06-08T18:57:36Z","display_at":"2026-06-08T18:57:36Z","incident_id":"01KTM9HWN7GW0M4Z4E1GXXYXE4","status":"investigating","updated_at":"2026-06-08T18:57:36Z"}]},{"id":"01KT4G3EJ32XHSVCA1GWWXC6Q5","name":"[EU] Elevated latencies in UI","status":"resolved","created_at":"2026-06-02T15:44:12Z","updated_at":"2026-06-02T16:55:53Z","resolved_at":"2026-06-02T16:55:53Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01KT4M6Q48MW955XFNWBN037QS","body":"This issue has been resolved.","created_at":"2026-06-02T16:55:53Z","display_at":"2026-06-02T16:55:53Z","incident_id":"01KT4G3EJ32XHSVCA1GWWXC6Q5","status":"resolved","updated_at":"2026-06-02T16:55:53Z"},{"id":"01KT4GTJ5FV9X7RB4XJGAG3K8Y","body":"We've changed infrastructure capacity and observe that this mitigates the problems. We continue to observe the situation.","created_at":"2026-06-02T15:56:49Z","display_at":"2026-06-02T15:56:49Z","incident_id":"01KT4G3EJ32XHSVCA1GWWXC6Q5","status":"monitoring","updated_at":"2026-06-02T15:56:49Z"},{"id":"01KT4G3EJ3Z6KP9V88R40ARNR5","body":"We're investigating elevated loading times within the Langfuse UI and our APIs.","created_at":"2026-06-02T15:44:12Z","display_at":"2026-06-02T15:44:12Z","incident_id":"01KT4G3EJ32XHSVCA1GWWXC6Q5","status":"investigating","updated_at":"2026-06-02T15:44:12Z"}]},{"id":"01KT1GH4K1EESQTRGS4RGBDGQ9","name":"[EU] Elevated error rates and delayed inserts","status":"resolved","created_at":"2026-06-01T11:53:57Z","updated_at":"2026-06-01T14:13:26Z","resolved_at":"2026-06-01T14:13:26Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01KT1RGHBYTYVETESVG4JMRJEF","body":"The underlying incident with our infrastructure (https://statuspage.incident.io/clickhousecloud/incidents/01KT1G25S9PBKM7VJEB146680G) has been resolved.","created_at":"2026-06-01T14:13:26Z","display_at":"2026-06-01T14:13:26Z","incident_id":"01KT1GH4K1EESQTRGS4RGBDGQ9","status":"resolved","updated_at":"2026-06-01T14:13:26Z"},{"id":"01KT1GH4K15JRMZNTBRA36XDA3","body":"We're investigating an issue with elevated latencies and error rates in the EU environment. ","created_at":"2026-06-01T11:53:57Z","display_at":"2026-06-01T11:53:57Z","incident_id":"01KT1GH4K1EESQTRGS4RGBDGQ9","status":"investigating","updated_at":"2026-06-01T11:53:57Z"}]},{"id":"01KSQNV81DX0PQQTSJX6FAM1E4","name":"[US] Delayed LLM as a judge evaluation","status":"resolved","created_at":"2026-05-28T16:14:27Z","updated_at":"2026-05-28T17:13:36Z","resolved_at":"2026-05-28T17:13:36Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01KSQS7HRS902NDAHD9YFGDGFS","body":"We've fully caught up and process LLM as a judge in realtime again.","created_at":"2026-05-28T17:13:36Z","display_at":"2026-05-28T17:13:36Z","incident_id":"01KSQNV81DX0PQQTSJX6FAM1E4","status":"resolved","updated_at":"2026-05-28T17:13:36Z"},{"id":"01KSQNV81D0G240MFA3ESM02BG","body":"We're currently investigating delays in our LLM as a judge execution times.","created_at":"2026-05-28T16:14:27Z","display_at":"2026-05-28T16:14:27Z","incident_id":"01KSQNV81DX0PQQTSJX6FAM1E4","status":"investigating","updated_at":"2026-05-28T16:14:27Z"}]},{"id":"01KSN72STM8HKPGS5TFGAKY3XK","name":"LLM as a Judge execution is delayed","status":"resolved","created_at":"2026-05-27T17:17:57Z","updated_at":"2026-05-27T18:36:20Z","resolved_at":"2026-05-27T18:36:20Z","impact":"minor","page_id":"01KS5BHY7AHH8P8EFW4ZENDP28","incident_updates":[{"id":"01KSNBJAZNFSZRRFKESWRWYHXN","body":"We have scaled our infra to drain the queue backlog. Evals are executed without delay again.","created_at":"2026-05-27T18:36:20Z","display_at":"2026-05-27T18:36:20Z","incident_id":"01KSN72STM8HKPGS5TFGAKY3XK","status":"resolved","updated_at":"2026-05-27T18:36:20Z"},{"id":"01KSN9K2F83PPXHFXS3Y3HS2Z6","body":"We continue to monitor the situation","created_at":"2026-05-27T18:01:47Z","display_at":"2026-05-27T18:01:47Z","incident_id":"01KSN72STM8HKPGS5TFGAKY3XK","status":"monitoring","updated_at":"2026-05-27T18:01:47Z"},{"id":"01KSN9JCEJFSBXT7EH07HKQBQH","body":"We believe only a single customer is affected and it is caused by LLM's provider rate limits. Since we monitor worst delay across our system this initially looked as a bigger issue.","created_at":"2026-05-27T18:01:25Z","display_at":"2026-05-27T18:01:25Z","incident_id":"01KSN72STM8HKPGS5TFGAKY3XK","status":"identified","updated_at":"2026-05-27T18:01:25Z"},{"id":"01KSN72STMB4V93SBBCGMYEN4G","body":"We are investigating the root cause","created_at":"2026-05-27T17:17:57Z","display_at":"2026-05-27T13:17:00Z","incident_id":"01KSN72STM8HKPGS5TFGAKY3XK","status":"investigating","updated_at":"2026-05-27T17:19:11Z"}]}]}