diff --git a/app/controllers/api/v1/cron_check_ins_controller.rb b/app/controllers/api/v1/cron_check_ins_controller.rb index c7df3fdc..a2facc80 100644 --- a/app/controllers/api/v1/cron_check_ins_controller.rb +++ b/app/controllers/api/v1/cron_check_ins_controller.rb @@ -55,7 +55,12 @@ def create end render json: { status: "ok", message: "Run started recorded" } when "error" - render json: { status: "ok", message: "Error acknowledged" } + ActsAsTenant.with_tenant(check_in.account) do + check_in.record_error_ping!(source_ip: request.remote_ip) + end + msg = params[:message].presence || params[:error_message].presence + Rails.logger.warn("[CronCheckIns] slug=#{slug} error from job#{msg ? ": #{msg.to_s.truncate(500)}" : ""}") + render json: { status: "ok", message: "Error recorded" } end rescue => e Rails.logger.error("[CronCheckIns] Error: #{e.message}") diff --git a/app/controllers/api/v1/replay_sessions_controller.rb b/app/controllers/api/v1/replay_sessions_controller.rb index bb45269c..b08ab2ee 100644 --- a/app/controllers/api/v1/replay_sessions_controller.rb +++ b/app/controllers/api/v1/replay_sessions_controller.rb @@ -1,45 +1,47 @@ class Api::V1::ReplaySessionsController < Api::BaseController # POST /api/v1/replay_sessions def create - ActsAsTenant.with_tenant(@current_project.account) do - raw_events_json = params[:events].to_json + unless Rails.env.development? + ActsAsTenant.with_tenant(@current_project.account) do + raw_events_json = params[:events].to_json - # Upsert: if replay_id already exists for this project, update it with new events - existing = @current_project.replays.find_by(replay_id: params[:replay_id]) + # Upsert: if replay_id already exists for this project, update it with new events + existing = @current_project.replays.find_by(replay_id: params[:replay_id]) - if existing - # Re-ingest with updated events - ReplayIngestJob.perform_async(existing.id, raw_events_json) - render json: { status: "updated", replay_id: existing.replay_id }, status: :accepted - return - end + if existing + # Re-ingest with updated events + ReplayIngestJob.perform_async(existing.id, raw_events_json) + render json: { status: "updated", replay_id: existing.replay_id }, status: :accepted + return + end - # New replay - if @current_project.account.replay_quota_exceeded? - render json: { error: "quota_exceeded", message: "Replay quota exceeded" }, status: :too_many_requests - return - end + # New replay + if @current_project.account.replay_quota_exceeded? + render json: { error: "quota_exceeded", message: "Replay quota exceeded" }, status: :too_many_requests + return + end - replay = Replay.new(replay_params) - replay.account = @current_project.account - replay.project = @current_project - replay.status = "pending" - replay.retention_until = 30.days.from_now + replay = Replay.new(replay_params) + replay.account = @current_project.account + replay.project = @current_project + replay.status = "pending" + replay.retention_until = 30.days.from_now - unless replay.save - render json: { - error: "validation_error", - message: replay.errors.full_messages.join(", ") - }, status: :unprocessable_entity - return - end + unless replay.save + render json: { + error: "validation_error", + message: replay.errors.full_messages.join(", ") + }, status: :unprocessable_entity + return + end - @current_project.account.increment_replay_usage! + @current_project.account.increment_replay_usage! - ReplayIngestJob.perform_async(replay.id, raw_events_json) - ReplayIssueLinkJob.perform_async(replay.id) + ReplayIngestJob.perform_async(replay.id, raw_events_json) + ReplayIssueLinkJob.perform_async(replay.id) - render json: { status: "accepted", replay_id: replay.replay_id }, status: :accepted + render json: { status: "accepted", replay_id: replay.replay_id }, status: :accepted + end end end diff --git a/app/controllers/check_ins_controller.rb b/app/controllers/check_ins_controller.rb index c87c7cc0..45c128d0 100644 --- a/app/controllers/check_ins_controller.rb +++ b/app/controllers/check_ins_controller.rb @@ -119,7 +119,7 @@ def calculate_uptime(check_in) expected_pings = (hours_since_creation * 3600 / interval).floor return 100.0 if expected_pings.zero? - actual_pings = check_in.pings.count + actual_pings = check_in.pings.where(status: "success").count [(actual_pings.to_f / expected_pings * 100).round(1), 100.0].min end @@ -139,20 +139,31 @@ def build_ping_timeline(check_in) cursor = 24.hours.ago.beginning_of_hour while cursor < Time.current slot_end = cursor + slot_duration - count = pings.count { |p| p.pinged_at >= cursor && p.pinged_at < slot_end } + in_slot = ->(p) { p.pinged_at >= cursor && p.pinged_at < slot_end } + count_success = pings.count { |p| in_slot.call(p) && p.status == "success" } + count_error = pings.count { |p| in_slot.call(p) && p.status == "error" } + count = count_success + count_error expected = slot_duration >= interval ? 1 : 0 - status = if check_in.last_seen_at.nil? && count.zero? + status = if check_in.last_seen_at.nil? && count_success.zero? && count_error.zero? "empty" - elsif count > 0 + elsif count_success > 0 "ok" + elsif count_error > 0 + "error" elsif expected > 0 && cursor < Time.current - interval "missed" else "pending" end - slots << { time: cursor, count: count, status: status } + slots << { + time: cursor, + count: count, + count_success: count_success, + count_error: count_error, + status: status + } cursor = slot_end end diff --git a/app/models/check_in.rb b/app/models/check_in.rb index 15abf648..2cd747dd 100644 --- a/app/models/check_in.rb +++ b/app/models/check_in.rb @@ -53,17 +53,46 @@ def record_success_ping!(source_ip: nil) ) end + # Failed run from the app (after :in_progress). Does not move last_seen_at / success clock. + def record_error_ping!(source_ip: nil) + update!( + run_started_at: nil, + last_status: "missed" + ) + pings.create!( + status: "error", + source_ip: source_ip, + pinged_at: Time.current + ) + end + def token read_attribute(:identifier) end def overdue? - return false unless enabled? && last_seen_at.present? && heartbeat_interval_seconds.present? + return false unless enabled? && heartbeat_interval_seconds.present? + + if run_started_at.present? && run_started_at < heartbeat_interval_seconds.seconds.ago + return true + end + + return false unless last_seen_at.present? last_seen_at < heartbeat_interval_seconds.seconds.ago end + # Latest row wins: error after success shows "error" until the next successful :ok/:success ping. def status_display + if run_started_at.present? + return "missed" if heartbeat_interval_seconds.present? && run_started_at < heartbeat_interval_seconds.seconds.ago + + return "running" + end + + latest = pings.order(pinged_at: :desc, id: :desc).first + return "error" if latest&.status == "error" + return "new" if last_seen_at.nil? return "missed" if overdue? @@ -74,6 +103,8 @@ def status_color case status_display when "healthy" then "green" when "missed" then "red" + when "error" then "red" + when "running" then "blue" when "new" then "gray" else "gray" end @@ -98,6 +129,13 @@ def ping_url def should_alert? return false unless enabled? + + latest = pings.order(pinged_at: :desc, id: :desc).first + if latest&.status == "error" + return false if last_alerted_at.present? && last_alerted_at >= latest.pinged_at + return true + end + return false unless overdue? return false if last_alerted_at.present? && last_alerted_at > 1.hour.ago diff --git a/app/models/check_in_ping.rb b/app/models/check_in_ping.rb index 310f0f3f..4b9e67ba 100644 --- a/app/models/check_in_ping.rb +++ b/app/models/check_in_ping.rb @@ -6,7 +6,7 @@ class CheckInPing < ApplicationRecord belongs_to :check_in validates :pinged_at, presence: true - validates :status, inclusion: { in: %w[success] } + validates :status, inclusion: { in: %w[success error] } scope :recent, -> { order(pinged_at: :desc) } scope :today, -> { where(pinged_at: Time.current.beginning_of_day..) } diff --git a/app/views/check_ins/index.html.erb b/app/views/check_ins/index.html.erb index 4b138c0b..3b5384d6 100644 --- a/app/views/check_ins/index.html.erb +++ b/app/views/check_ins/index.html.erb @@ -49,6 +49,8 @@ badge_class = case check_in.status_display when "healthy" then "bg-green-100 text-green-800" when "missed" then "bg-red-100 text-red-800" + when "error" then "bg-amber-100 text-amber-900" + when "running" then "bg-blue-100 text-blue-800" when "new" then "bg-gray-100 text-gray-600" else "bg-gray-100 text-gray-600" end diff --git a/app/views/check_ins/show.html.erb b/app/views/check_ins/show.html.erb index a9227ae3..606d4ffe 100644 --- a/app/views/check_ins/show.html.erb +++ b/app/views/check_ins/show.html.erb @@ -13,6 +13,8 @@ badge_class = case @check_in.status_display when "healthy" then "bg-green-100 text-green-800" when "missed" then "bg-red-100 text-red-800" + when "error" then "bg-amber-100 text-amber-900" + when "running" then "bg-blue-100 text-blue-800" when "new" then "bg-gray-100 text-gray-600" else "bg-gray-100 text-gray-600" end @@ -79,6 +81,7 @@

Last 24 Hours

OK + Job error Missed Pending
@@ -88,14 +91,19 @@ <% bar_color = case slot[:status] when "ok" then "bg-green-500 hover:bg-green-600" + when "error" then "bg-amber-500 hover:bg-amber-600" when "missed" then "bg-red-500 hover:bg-red-600" when "pending" then "bg-gray-200 hover:bg-gray-300" else "bg-gray-100" end bar_height = slot[:status] == "ok" ? "h-full" : "h-full" + tip_parts = [] + tip_parts << "#{slot[:count_success].to_i} OK" if slot[:count_success].to_i.positive? + tip_parts << "#{slot[:count_error].to_i} err" if slot[:count_error].to_i.positive? + tip_label = tip_parts.any? ? tip_parts.join(", ") : "#{slot[:count]} event#{slot[:count] != 1 ? 's' : ''}" %>
+ title="<%= slot[:time].strftime('%H:%M') %> — <%= tip_label %> (<%= slot[:status] %>)">
<% end %> @@ -211,10 +219,17 @@ -

After the task or job succeeds, report OK (from Ruby):

- 
ActiveRabbit::Client.capture_cron_check_in("<%= @check_in.slug %>", :ok)
+

Statuses should come from your cron job, not from an external schedule guessing when it runs:

+ + 
ActiveRabbit::Client.capture_cron_check_in("<%= @check_in.slug %>", :in_progress)
+# ... work ...
+ActiveRabbit::Client.capture_cron_check_in("<%= @check_in.slug %>", :ok)
-

Optional statuses: :in_progress at start, :error in rescue. For Active Job you can use:

+

For Active Job you can use:

class MyCronJob <%= "<" %> ApplicationJob
   include ActiveRabbit::Client::CronMonitor
   active_rabbit_cron slug: "<%= @check_in.slug %>"
diff --git a/app/views/uptime/monitors/index.html.erb b/app/views/uptime/monitors/index.html.erb
index e256142b..9f8dc34a 100644
--- a/app/views/uptime/monitors/index.html.erb
+++ b/app/views/uptime/monitors/index.html.erb
@@ -65,6 +65,8 @@
                     badge_class = case check_in.status_display
                                   when "healthy" then "bg-green-100 text-green-800"
                                   when "missed" then "bg-red-100 text-red-800"
+                                  when "error" then "bg-amber-100 text-amber-900"
+                                  when "running" then "bg-blue-100 text-blue-800"
                                   when "new" then "bg-gray-100 text-gray-600"
                                   else "bg-gray-100 text-gray-600"
                                   end
diff --git a/test/integration/api_check_ins_test.rb b/test/integration/api_check_ins_test.rb
index 9ff374a2..a6cccc58 100644
--- a/test/integration/api_check_ins_test.rb
+++ b/test/integration/api_check_ins_test.rb
@@ -89,13 +89,18 @@ class ApiCheckInsTest < ActionDispatch::IntegrationTest
     assert_nil @check_in.run_started_at
   end
 
-  test "POST cron check_ins error does not record ping" do
-    assert_no_difference -> { CheckInPing.where(check_in_id: @check_in.id).count } do
+  test "POST cron check_ins error records error ping and optional message" do
+    assert_difference -> { CheckInPing.where(check_in_id: @check_in.id, status: "error").count }, 1 do
       post "/api/v1/cron/check_ins",
-           params: { slug: @check_in.slug, status: "error" }.to_json,
+           params: { slug: @check_in.slug, status: "error", message: "job failed" }.to_json,
            headers: @api_headers
     end
     assert_response :success
+    json = JSON.parse(response.body)
+    assert_equal "ok", json["status"]
+    @check_in.reload
+    assert_nil @check_in.run_started_at
+    assert_equal "missed", @check_in.last_status
   end
 
   test "POST cron check_ins missing slug is bad_request" do
diff --git a/test/models/check_in_test.rb b/test/models/check_in_test.rb
index 193f1417..949d5c49 100644
--- a/test/models/check_in_test.rb
+++ b/test/models/check_in_test.rb
@@ -136,4 +136,33 @@ class CheckInTest < ActiveSupport::TestCase
     assert ci.last_seen_at.present?
     assert_equal "127.0.0.1", ci.pings.last.source_ip
   end
+
+  test "record_error_ping! creates error row without moving last_seen_at" do
+    ci = check_ins(:healthy)
+    had_seen = ci.last_seen_at
+
+    assert_difference -> { ci.pings.where(status: "error").count }, 1 do
+      ActsAsTenant.with_tenant(@account) do
+        ci.record_error_ping!(source_ip: "10.0.0.1")
+      end
+    end
+
+    ci.reload
+    assert_equal had_seen.to_i, ci.last_seen_at.to_i
+    assert_equal "error", ci.pings.order(pinged_at: :desc).first.status
+    assert_equal "error", ci.status_display
+  end
+
+  test "status_display running when run_started_at fresh" do
+    ci = check_ins(:healthy)
+    ci.update_columns(run_started_at: 30.seconds.ago)
+    assert_equal "running", ci.status_display
+  end
+
+  test "should_alert? true when latest ping is error and not yet alerted" do
+    ci = check_ins(:healthy)
+    ActsAsTenant.with_tenant(@account) { ci.record_error_ping! }
+    ci.update_column(:last_alerted_at, nil)
+    assert ci.should_alert?
+  end
 end