Add Maze Runner tests for 'on error' callbacks

features/fixtures/docker-compose.yml

@@ -120,6 +120,7 @@ services:
       - BUGSNAG_TIMEOUT
       - CALLBACK_INITIATOR
       - SQL_ONLY_BREADCRUMBS
+      - ADD_ON_ERROR
       - USE_DEFAULT_AUTO_CAPTURE_SESSIONS
     restart: "no"
 
@@ -155,6 +156,7 @@ services:
       - BUGSNAG_TIMEOUT
       - CALLBACK_INITIATOR
       - SQL_ONLY_BREADCRUMBS
+      - ADD_ON_ERROR
       - USE_DEFAULT_AUTO_CAPTURE_SESSIONS
     restart: "no"
 
@@ -190,6 +192,7 @@ services:
       - BUGSNAG_TIMEOUT
       - CALLBACK_INITIATOR
       - SQL_ONLY_BREADCRUMBS
+      - ADD_ON_ERROR
       - USE_DEFAULT_AUTO_CAPTURE_SESSIONS
     restart: "no"
 
@@ -225,6 +228,7 @@ services:
       - BUGSNAG_TIMEOUT
       - CALLBACK_INITIATOR
       - SQL_ONLY_BREADCRUMBS
+      - ADD_ON_ERROR
       - USE_DEFAULT_AUTO_CAPTURE_SESSIONS
     restart: "no"
     networks:
@@ -296,4 +300,4 @@ services:
 
 networks:
   default:
-    name: ${NETWORK_NAME}
+    name: ${NETWORK_NAME}

features/fixtures/plain/app/report_modification/initiators/handled_on_error.rb

@@ -0,0 +1,10 @@
+require 'bugsnag'
+require './app'
+
+configure_basics
+
+def run(callback)
+  Bugsnag.add_on_error(callback)
+
+  Bugsnag.notify(RuntimeError.new("Oh no"))
+end

features/fixtures/plain/app/report_modification/initiators/unhandled_on_error.rb

@@ -0,0 +1,11 @@
+require 'bugsnag'
+require './app'
+
+configure_basics
+add_at_exit
+
+def run(callback)
+  Bugsnag.add_on_error(callback)
+
+  raise RuntimeError.new "Oh no"
+end

features/fixtures/plain/app/stack_frame_modification/initiators/handled_on_error.rb

@@ -0,0 +1,29 @@
+require 'bugsnag'
+require './app'
+
+configure_basics
+
+def run(callback)
+  Bugsnag.add_on_error(callback)
+  step_one
+end
+
+def step_one
+  step_two
+end
+
+def step_two
+  step_three
+end
+
+def step_three
+  crash
+end
+
+def crash
+  begin
+    "Test".insrt(-1, "!")
+  rescue Exception => e
+    Bugsnag.notify(e)
+  end
+end

features/fixtures/plain/app/stack_frame_modification/initiators/unhandled_on_error.rb

@@ -0,0 +1,26 @@
+require 'bugsnag'
+require './app'
+
+configure_basics
+add_at_exit
+
+def run(callback)
+  Bugsnag.add_on_error(callback)
+  step_one
+end
+
+def step_one
+  step_two
+end
+
+def step_two
+  step_three
+end
+
+def step_three
+  crash
+end
+
+def crash
+  raise RuntimeError.new "Oh no"
+end

features/fixtures/rails3/app/config/initializers/bugsnag.rb

@@ -18,4 +18,12 @@
       breadcrumb.ignore! unless breadcrumb.meta_data[:event_name] == "sql.active_record" && breadcrumb.meta_data[:name] == "User Load"
     end
   end
+
+  if ENV["ADD_ON_ERROR"] == "true"
+    config.add_on_error(proc do |report|
+      report.add_tab(:on_error, {
+        source: report.unhandled ? 'on_error unhandled' : 'on_error handled'
+      })
+    end)
+  end
 end

features/fixtures/rails4/app/config/initializers/bugsnag.rb

@@ -18,4 +18,12 @@
       breadcrumb.ignore! unless breadcrumb.meta_data[:event_name] == "sql.active_record" && breadcrumb.meta_data[:name] == "User Load"
     end
   end
+
+  if ENV["ADD_ON_ERROR"] == "true"
+    config.add_on_error(proc do |report|
+      report.add_tab(:on_error, {
+        source: report.unhandled ? 'on_error unhandled' : 'on_error handled'
+      })
+    end)
+  end
 end

features/fixtures/rails5/app/config/initializers/bugsnag.rb

@@ -18,4 +18,12 @@
       breadcrumb.ignore! unless breadcrumb.meta_data[:event_name] == "sql.active_record" && breadcrumb.meta_data[:name] == "User Load"
     end
   end
+
+  if ENV["ADD_ON_ERROR"] == "true"
+    config.add_on_error(proc do |report|
+      report.add_tab(:on_error, {
+        source: report.unhandled ? 'on_error unhandled' : 'on_error handled'
+      })
+    end)
+  end
 end

features/fixtures/rails6/app/config/initializers/bugsnag.rb

@@ -18,4 +18,12 @@
       breadcrumb.ignore! unless breadcrumb.meta_data[:event_name] == "sql.active_record" && breadcrumb.meta_data[:name] == "User Load"
     end
   end
+
+  if ENV["ADD_ON_ERROR"] == "true"
+    config.add_on_error(proc do |report|
+      report.add_tab(:on_error, {
+        source: report.unhandled ? 'on_error unhandled' : 'on_error handled'
+      })
+    end)
+  end
 end

features/plain_features/add_tab.feature

@@ -15,6 +15,8 @@ Scenario Outline: Metadata can be added to a report using add_tab
   | handled_before_notify   |
   | handled_block           |
   | unhandled_before_notify |
+  | handled_on_error        |
+  | unhandled_on_error      |
 
 Scenario Outline: Metadata can be added to an existing tab using add_tab
   Given I set environment variable "CALLBACK_INITIATOR" to "<initiator>"
@@ -33,6 +35,8 @@ Scenario Outline: Metadata can be added to an existing tab using add_tab
   | handled_before_notify   |
   | handled_block           |
   | unhandled_before_notify |
+  | handled_on_error        |
+  | unhandled_on_error      |
 
 Scenario Outline: Metadata can be overwritten using add_tab
   Given I set environment variable "CALLBACK_INITIATOR" to "<initiator>"
@@ -46,4 +50,6 @@ Scenario Outline: Metadata can be overwritten using add_tab
   | initiator               |
   | handled_before_notify   |
   | handled_block           |
-  | unhandled_before_notify |
+  | unhandled_before_notify |
+  | handled_on_error        |
+  | unhandled_on_error      |

features/plain_features/ignore_report.feature

@@ -10,3 +10,5 @@ Scenario Outline: A reports severity can be modified
   | initiator               |
   | handled_before_notify   |
   | unhandled_before_notify |
+  | handled_on_error        |
+  | unhandled_on_error      |

features/plain_features/report_api_key.feature

@@ -11,4 +11,6 @@ Scenario Outline: A report can have its api_key modified
   | initiator               |
   | handled_before_notify   |
   | handled_block           |
-  | unhandled_before_notify |
+  | unhandled_before_notify |
+  | handled_on_error        |
+  | unhandled_on_error      |

features/plain_features/report_severity.feature

@@ -13,3 +13,5 @@ Scenario Outline: A reports severity can be modified
   | handled_before_notify   |
   | handled_block           |
   | unhandled_before_notify |
+  | handled_on_error        |
+  | unhandled_on_error      |

features/plain_features/report_stack_frames.feature

@@ -12,6 +12,8 @@ Scenario Outline: Stack frames can be removed
   | initiator               | lineNumber |
   | handled_before_notify   | 20         |
   | unhandled_before_notify | 21         |
+  | handled_on_error        | 20         |
+  | unhandled_on_error      | 21         |
 
 Scenario: Stack frames can be removed from a notified string
   Given I set environment variable "CALLBACK_INITIATOR" to "handled_block"
@@ -36,6 +38,8 @@ Scenario Outline: Stack frames can be marked as in project
   | initiator               |
   | handled_before_notify   |
   | unhandled_before_notify |
+  | handled_on_error        |
+  | unhandled_on_error      |
 
 Scenario: Stack frames can be marked as in project with a handled string
   Given I set environment variable "CALLBACK_INITIATOR" to "handled_block"

features/plain_features/report_user.feature

@@ -14,6 +14,8 @@ Scenario Outline: A report can have a user name, email, and id set
   | handled_before_notify   |
   | handled_block           |
   | unhandled_before_notify |
+  | handled_on_error        |
+  | unhandled_on_error      |
 
 Scenario Outline: A report can have custom info set
   Given I set environment variable "CALLBACK_INITIATOR" to "<initiator>"
@@ -30,6 +32,8 @@ Scenario Outline: A report can have custom info set
   | handled_before_notify   |
   | handled_block           |
   | unhandled_before_notify |
+  | handled_on_error        |
+  | unhandled_on_error      |
 
 Scenario Outline: A report can have its user info removed
   Given I set environment variable "CALLBACK_INITIATOR" to "<initiator>"
@@ -42,4 +46,6 @@ Scenario Outline: A report can have its user info removed
   | initiator               |
   | handled_before_notify   |
   | handled_block           |
-  | unhandled_before_notify |
+  | unhandled_before_notify |
+  | handled_on_error        |
+  | unhandled_on_error      |

features/rails_features/on_error.feature

@@ -0,0 +1,29 @@
+Feature: On error callbacks
+
+@rails3 @rails4 @rails5 @rails6
+Scenario: Rails on_error works on handled errors
+  Given I set environment variable "ADD_ON_ERROR" to "true"
+  And I start the rails service
+  When I navigate to the route "/handled/unthrown" on the rails app
+  And I wait to receive a request
+  Then the request is valid for the error reporting API version "4.0" for the "Ruby Bugsnag Notifier"
+  And the exception "errorClass" equals "RuntimeError"
+  And the exception "message" starts with "handled unthrown error"
+  And the event "unhandled" is false
+  And the event "app.type" equals "rails"
+  And the event "metaData.request.url" ends with "/handled/unthrown"
+  And the event "metaData.on_error.source" equals "on_error handled"
+
+@rails3 @rails4 @rails5 @rails6
+Scenario: Rails on_error works on unhandled errors
+  Given I set environment variable "ADD_ON_ERROR" to "true"
+  And I start the rails service
+  When I navigate to the route "/unhandled/error" on the rails app
+  And I wait to receive a request
+  Then the request is valid for the error reporting API version "4.0" for the "Ruby Bugsnag Notifier"
+  And the exception "errorClass" equals "NameError"
+  And the exception "message" starts with "undefined local variable or method `generate_unhandled_error' for #<UnhandledController"
+  And the event "unhandled" is true
+  And the event "app.type" equals "rails"
+  And the event "metaData.request.url" ends with "/unhandled/error"
+  And the event "metaData.on_error.source" equals "on_error unhandled"