feat(check-ins): enhance check-in error handling and status reporting

app/controllers/api/v1/cron_check_ins_controller.rb

@@ -55,7 +55,12 @@ def create
           end
           render json: { status: "ok", message: "Run started recorded" }
         when "error"
-          render json: { status: "ok", message: "Error acknowledged" }
+          ActsAsTenant.with_tenant(check_in.account) do
+            check_in.record_error_ping!(source_ip: request.remote_ip)
+          end
+          msg = params[:message].presence || params[:error_message].presence
+          Rails.logger.warn("[CronCheckIns] slug=#{slug} error from job#{msg ? ": #{msg.to_s.truncate(500)}" : ""}")
+          render json: { status: "ok", message: "Error recorded" }
         end
       rescue => e
         Rails.logger.error("[CronCheckIns] Error: #{e.message}")

app/controllers/api/v1/replay_sessions_controller.rb

@@ -1,45 +1,47 @@
 class Api::V1::ReplaySessionsController < Api::BaseController
   # POST /api/v1/replay_sessions
   def create
-    ActsAsTenant.with_tenant(@current_project.account) do
-      raw_events_json = params[:events].to_json
+    unless Rails.env.development?
+      ActsAsTenant.with_tenant(@current_project.account) do
+        raw_events_json = params[:events].to_json
 
-      # Upsert: if replay_id already exists for this project, update it with new events
-      existing = @current_project.replays.find_by(replay_id: params[:replay_id])
+        # Upsert: if replay_id already exists for this project, update it with new events
+        existing = @current_project.replays.find_by(replay_id: params[:replay_id])
 
-      if existing
-        # Re-ingest with updated events
-        ReplayIngestJob.perform_async(existing.id, raw_events_json)
-        render json: { status: "updated", replay_id: existing.replay_id }, status: :accepted
-        return
-      end
+        if existing
+          # Re-ingest with updated events
+          ReplayIngestJob.perform_async(existing.id, raw_events_json)
+          render json: { status: "updated", replay_id: existing.replay_id }, status: :accepted
+          return
+        end
 
-      # New replay
-      if @current_project.account.replay_quota_exceeded?
-        render json: { error: "quota_exceeded", message: "Replay quota exceeded" }, status: :too_many_requests
-        return
-      end
+        # New replay
+        if @current_project.account.replay_quota_exceeded?
+          render json: { error: "quota_exceeded", message: "Replay quota exceeded" }, status: :too_many_requests
+          return
+        end
 
-      replay = Replay.new(replay_params)
-      replay.account = @current_project.account
-      replay.project = @current_project
-      replay.status = "pending"
-      replay.retention_until = 30.days.from_now
+        replay = Replay.new(replay_params)
+        replay.account = @current_project.account
+        replay.project = @current_project
+        replay.status = "pending"
+        replay.retention_until = 30.days.from_now
 
-      unless replay.save
-        render json: {
-          error: "validation_error",
-          message: replay.errors.full_messages.join(", ")
-        }, status: :unprocessable_entity
-        return
-      end
+        unless replay.save
+          render json: {
+            error: "validation_error",
+            message: replay.errors.full_messages.join(", ")
+          }, status: :unprocessable_entity
+          return
+        end
 
-      @current_project.account.increment_replay_usage!
+        @current_project.account.increment_replay_usage!
 
-      ReplayIngestJob.perform_async(replay.id, raw_events_json)
-      ReplayIssueLinkJob.perform_async(replay.id)
+        ReplayIngestJob.perform_async(replay.id, raw_events_json)
+        ReplayIssueLinkJob.perform_async(replay.id)
 
-      render json: { status: "accepted", replay_id: replay.replay_id }, status: :accepted
+        render json: { status: "accepted", replay_id: replay.replay_id }, status: :accepted
+      end
     end
   end
 

app/controllers/check_ins_controller.rb

@@ -119,7 +119,7 @@ def calculate_uptime(check_in)
     expected_pings = (hours_since_creation * 3600 / interval).floor
     return 100.0 if expected_pings.zero?
 
-    actual_pings = check_in.pings.count
+    actual_pings = check_in.pings.where(status: "success").count
     [(actual_pings.to_f / expected_pings * 100).round(1), 100.0].min
   end
 
@@ -139,20 +139,31 @@ def build_ping_timeline(check_in)
     cursor = 24.hours.ago.beginning_of_hour
     while cursor < Time.current
       slot_end = cursor + slot_duration
-      count = pings.count { |p| p.pinged_at >= cursor && p.pinged_at < slot_end }
+      in_slot = ->(p) { p.pinged_at >= cursor && p.pinged_at < slot_end }
+      count_success = pings.count { |p| in_slot.call(p) && p.status == "success" }
+      count_error = pings.count { |p| in_slot.call(p) && p.status == "error" }
+      count = count_success + count_error
       expected = slot_duration >= interval ? 1 : 0
 
-      status = if check_in.last_seen_at.nil? && count.zero?
+      status = if check_in.last_seen_at.nil? && count_success.zero? && count_error.zero?
                  "empty"
-      elsif count > 0
+      elsif count_success > 0
                  "ok"
+      elsif count_error > 0
+                 "error"
       elsif expected > 0 && cursor < Time.current - interval
                  "missed"
       else
                  "pending"
       end
 
-      slots << { time: cursor, count: count, status: status }
+      slots << {
+        time: cursor,
+        count: count,
+        count_success: count_success,
+        count_error: count_error,
+        status: status
+      }
       cursor = slot_end
     end
 

app/models/check_in.rb

@@ -53,17 +53,46 @@ def record_success_ping!(source_ip: nil)
     )
   end
 
+  # Failed run from the app (after :in_progress). Does not move last_seen_at / success clock.
+  def record_error_ping!(source_ip: nil)
+    update!(
+      run_started_at: nil,
+      last_status: "missed"
+    )
+    pings.create!(
+      status: "error",
+      source_ip: source_ip,
+      pinged_at: Time.current
+    )
+  end
+
   def token
     read_attribute(:identifier)
   end
 
   def overdue?
-    return false unless enabled? && last_seen_at.present? && heartbeat_interval_seconds.present?
+    return false unless enabled? && heartbeat_interval_seconds.present?
+
+    if run_started_at.present? && run_started_at < heartbeat_interval_seconds.seconds.ago
+      return true
+    end
+
+    return false unless last_seen_at.present?
 
     last_seen_at < heartbeat_interval_seconds.seconds.ago
   end
 
+  # Latest row wins: error after success shows "error" until the next successful :ok/:success ping.
   def status_display
+    if run_started_at.present?
+      return "missed" if heartbeat_interval_seconds.present? && run_started_at < heartbeat_interval_seconds.seconds.ago
+
+      return "running"
+    end
+
+    latest = pings.order(pinged_at: :desc, id: :desc).first
+    return "error" if latest&.status == "error"
+
     return "new" if last_seen_at.nil?
     return "missed" if overdue?
 
@@ -74,6 +103,8 @@ def status_color
     case status_display
     when "healthy" then "green"
     when "missed" then "red"
+    when "error" then "red"
+    when "running" then "blue"
     when "new" then "gray"
     else "gray"
     end
@@ -98,6 +129,13 @@ def ping_url
 
   def should_alert?
     return false unless enabled?
+
+    latest = pings.order(pinged_at: :desc, id: :desc).first
+    if latest&.status == "error"
+      return false if last_alerted_at.present? && last_alerted_at >= latest.pinged_at
+      return true
+    end
+
     return false unless overdue?
     return false if last_alerted_at.present? && last_alerted_at > 1.hour.ago
 

app/models/check_in_ping.rb

@@ -6,7 +6,7 @@ class CheckInPing < ApplicationRecord
   belongs_to :check_in
 
   validates :pinged_at, presence: true
-  validates :status, inclusion: { in: %w[success] }
+  validates :status, inclusion: { in: %w[success error] }
 
   scope :recent, -> { order(pinged_at: :desc) }
   scope :today, -> { where(pinged_at: Time.current.beginning_of_day..) }

app/views/check_ins/index.html.erb

@@ -49,6 +49,8 @@
                   badge_class = case check_in.status_display
                                 when "healthy" then "bg-green-100 text-green-800"
                                 when "missed" then "bg-red-100 text-red-800"
+                                when "error" then "bg-amber-100 text-amber-900"
+                                when "running" then "bg-blue-100 text-blue-800"
                                 when "new" then "bg-gray-100 text-gray-600"
                                 else "bg-gray-100 text-gray-600"
                                 end

app/views/check_ins/show.html.erb

@@ -13,6 +13,8 @@
               badge_class = case @check_in.status_display
                             when "healthy" then "bg-green-100 text-green-800"
                             when "missed" then "bg-red-100 text-red-800"
+                            when "error" then "bg-amber-100 text-amber-900"
+                            when "running" then "bg-blue-100 text-blue-800"
                             when "new" then "bg-gray-100 text-gray-600"
                             else "bg-gray-100 text-gray-600"
                             end
@@ -79,6 +81,7 @@
       <h2 class="text-base sm:text-lg font-semibold text-gray-900">Last 24 Hours</h2>
       <div class="flex items-center gap-4 text-xs text-gray-500">
         <span class="flex items-center gap-1"><span class="w-2.5 h-2.5 rounded-sm bg-green-500 inline-block"></span> OK</span>
+        <span class="flex items-center gap-1"><span class="w-2.5 h-2.5 rounded-sm bg-amber-500 inline-block"></span> Job error</span>
         <span class="flex items-center gap-1"><span class="w-2.5 h-2.5 rounded-sm bg-red-500 inline-block"></span> Missed</span>
         <span class="flex items-center gap-1"><span class="w-2.5 h-2.5 rounded-sm bg-gray-200 inline-block"></span> Pending</span>
       </div>
@@ -88,14 +91,19 @@
         <%
           bar_color = case slot[:status]
                       when "ok" then "bg-green-500 hover:bg-green-600"
+                      when "error" then "bg-amber-500 hover:bg-amber-600"
                       when "missed" then "bg-red-500 hover:bg-red-600"
                       when "pending" then "bg-gray-200 hover:bg-gray-300"
                       else "bg-gray-100"
                       end
           bar_height = slot[:status] == "ok" ? "h-full" : "h-full"
+          tip_parts = []
+          tip_parts << "#{slot[:count_success].to_i} OK" if slot[:count_success].to_i.positive?
+          tip_parts << "#{slot[:count_error].to_i} err" if slot[:count_error].to_i.positive?
+          tip_label = tip_parts.any? ? tip_parts.join(", ") : "#{slot[:count]} event#{slot[:count] != 1 ? 's' : ''}"
         %>
         <div class="flex-1 <%= bar_height %> <%= bar_color %> rounded-sm transition-colors cursor-default"
-             title="<%= slot[:time].strftime('%H:%M') %> — <%= slot[:count] %> ping<%= slot[:count] != 1 ? 's' : '' %> (<%= slot[:status] %>)">
+             title="<%= slot[:time].strftime('%H:%M') %> — <%= tip_label %> (<%= slot[:status] %>)">
         </div>
       <% end %>
     </div>
@@ -211,10 +219,17 @@
         </button>
       </div>
 
-      <p class="text-sm text-gray-600 mb-2">After the task or job <span class="font-medium">succeeds</span>, report OK (from Ruby):</p>
-      <pre class="text-xs font-mono text-gray-800 bg-white border border-gray-200 rounded-lg p-3 overflow-x-auto whitespace-pre-wrap mb-4">ActiveRabbit::Client.capture_cron_check_in("<%= @check_in.slug %>", :ok)</pre>
+      <p class="text-sm text-gray-600 mb-2">Statuses should come from <span class="font-medium">your cron job</span>, not from an external schedule guessing when it runs:</p>
+      <ul class="list-disc list-inside text-sm text-gray-700 space-y-1 mb-3">
+        <li><code class="text-xs font-mono bg-gray-100 px-1 rounded">:in_progress</code> when the job <span class="font-medium">starts</span></li>
+        <li><code class="text-xs font-mono bg-gray-100 px-1 rounded">:ok</code> (or <code class="text-xs font-mono bg-gray-100 px-1 rounded">:success</code>) when it <span class="font-medium">finishes successfully</span></li>
+        <li><code class="text-xs font-mono bg-gray-100 px-1 rounded">:error</code> in <code class="font-mono">rescue</code> or after a failed run (optional JSON <code class="font-mono">message</code> on the same API)</li>
+      </ul>
+      <pre class="text-xs font-mono text-gray-800 bg-white border border-gray-200 rounded-lg p-3 overflow-x-auto whitespace-pre-wrap mb-4">ActiveRabbit::Client.capture_cron_check_in("<%= @check_in.slug %>", :in_progress)
+# ... work ...
+ActiveRabbit::Client.capture_cron_check_in("<%= @check_in.slug %>", :ok)</pre>
 
-      <p class="text-xs text-gray-500 mb-2">Optional statuses: <code class="font-mono">:in_progress</code> at start, <code class="font-mono">:error</code> in <code class="font-mono">rescue</code>. For Active Job you can use:</p>
+      <p class="text-xs text-gray-500 mb-2">For Active Job you can use:</p>
       <pre class="text-xs font-mono text-gray-800 bg-white border border-gray-200 rounded-lg p-3 overflow-x-auto whitespace-pre-wrap">class MyCronJob <%= "<" %> ApplicationJob
   include ActiveRabbit::Client::CronMonitor
   active_rabbit_cron slug: "<%= @check_in.slug %>"

app/views/uptime/monitors/index.html.erb

@@ -65,6 +65,8 @@
                     badge_class = case check_in.status_display
                                   when "healthy" then "bg-green-100 text-green-800"
                                   when "missed" then "bg-red-100 text-red-800"
+                                  when "error" then "bg-amber-100 text-amber-900"
+                                  when "running" then "bg-blue-100 text-blue-800"
                                   when "new" then "bg-gray-100 text-gray-600"
                                   else "bg-gray-100 text-gray-600"
                                   end

test/integration/api_check_ins_test.rb

@@ -89,13 +89,18 @@ class ApiCheckInsTest < ActionDispatch::IntegrationTest
     assert_nil @check_in.run_started_at
   end
 
-  test "POST cron check_ins error does not record ping" do
-    assert_no_difference -> { CheckInPing.where(check_in_id: @check_in.id).count } do
+  test "POST cron check_ins error records error ping and optional message" do
+    assert_difference -> { CheckInPing.where(check_in_id: @check_in.id, status: "error").count }, 1 do
       post "/api/v1/cron/check_ins",
-           params: { slug: @check_in.slug, status: "error" }.to_json,
+           params: { slug: @check_in.slug, status: "error", message: "job failed" }.to_json,
            headers: @api_headers
     end
     assert_response :success
+    json = JSON.parse(response.body)
+    assert_equal "ok", json["status"]
+    @check_in.reload
+    assert_nil @check_in.run_started_at
+    assert_equal "missed", @check_in.last_status
   end
 
   test "POST cron check_ins missing slug is bad_request" do

test/models/check_in_test.rb

@@ -136,4 +136,33 @@ class CheckInTest < ActiveSupport::TestCase
     assert ci.last_seen_at.present?
     assert_equal "127.0.0.1", ci.pings.last.source_ip
   end
+
+  test "record_error_ping! creates error row without moving last_seen_at" do
+    ci = check_ins(:healthy)
+    had_seen = ci.last_seen_at
+
+    assert_difference -> { ci.pings.where(status: "error").count }, 1 do
+      ActsAsTenant.with_tenant(@account) do
+        ci.record_error_ping!(source_ip: "10.0.0.1")
+      end
+    end
+
+    ci.reload
+    assert_equal had_seen.to_i, ci.last_seen_at.to_i
+    assert_equal "error", ci.pings.order(pinged_at: :desc).first.status
+    assert_equal "error", ci.status_display
+  end
+
+  test "status_display running when run_started_at fresh" do
+    ci = check_ins(:healthy)
+    ci.update_columns(run_started_at: 30.seconds.ago)
+    assert_equal "running", ci.status_display
+  end
+
+  test "should_alert? true when latest ping is error and not yet alerted" do
+    ci = check_ins(:healthy)
+    ActsAsTenant.with_tenant(@account) { ci.record_error_ping! }
+    ci.update_column(:last_alerted_at, nil)
+    assert ci.should_alert?
+  end
 end