Merge pull request #103 from geeksam/add-testing-hooks

Add testing hooks

Author: zerowidth

README.md

@@ -206,6 +206,8 @@ class MyExperiment
 end
 ```
 
+Note that the `#clean` method will discard the previous cleaner block if you call it again.  If for some reason you need to access the currently configured cleaner block, `Scientist::Experiment#cleaner` will return the block without further ado.  _(This probably won't come up in normal usage, but comes in handy if you're writing, say, a custom experiment runner that provides default cleaners.)_
+
 ### Ignoring mismatches
 
 During the early stages of an experiment, it's possible that some of your code will always generate a mismatch for reasons you know and understand but haven't yet fixed. Instead of these known cases always showing up as mismatches in your metrics or analysis, you can tell an experiment whether or not to ignore a mismatch using the `ignore` method. You may include more than one block if needed:
@@ -491,6 +493,22 @@ science "various-ways", run: "first-way" do |e|
 end
 ```
 
+#### Providing fake timing data
+
+If you're writing tests that depend on specific timing values, you can provide canned durations using the `fabricate_durations_for_testing_purposes` method, and Scientist will report these in `Scientist::Observation#duration` instead of the actual execution times.
+
+```ruby
+science "absolutely-nothing-suspicious-happening-here" do |e|
+  e.use { ... } # "control"
+  e.try { ... } # "candidate"
+  e.fabricate_durations_for_testing_purposes( "control" => 1.0, "candidate" => 0.5 )
+end
+```
+
+`fabricate_durations_for_testing_purposes` takes a Hash of duration values, keyed by behavior names.  (By default, Scientist uses `"control"` and `"candidate"`, but if you override these as shown in [Trying more than one thing](#trying-more-than-one-thing) or [No control, just candidates](#no-control-just-candidates), use matching names here.)  If a name is not provided, the actual execution time will be reported instead.
+
+_Like `Scientist::Experiment#cleaner`, this probably won't come up in normal usage.  It's here to make it easier to test code that extends Scientist._
+
 ### Without including Scientist
 
 If you need to use Scientist in a place where you aren't able to include the Scientist module, you can call `Scientist.run`:

doc/changelog.md

@@ -1,5 +1,10 @@
 # Changes
 
+## v1.2.1 (UNRELEASED)
+
+- New: Add an accessor for the configured clean block
+- New: Add a hook to use fabricated durations instead of actual timing data.
+
 ## v1.2.0 (5 July 2018)
 
 - New: Use monotonic clock for duration calculations

lib/scientist/experiment.rb

@@ -96,6 +96,13 @@ def clean(&block)
     @_scientist_cleaner = block
   end
 
+  # Accessor for the clean block, if one is available.
+  #
+  # Returns the configured block, or nil.
+  def cleaner
+    @_scientist_cleaner
+  end
+
   # Internal: Clean a value with the configured clean block, or return the value
   # if no clean block is configured.
   #
@@ -213,7 +220,8 @@ def run(name = nil)
 
     behaviors.keys.shuffle.each do |key|
       block = behaviors[key]
-      observations << Scientist::Observation.new(key, self, &block)
+      fabricated_duration = @_scientist_fabricated_durations && @_scientist_fabricated_durations[key]
+      observations << Scientist::Observation.new(key, self, fabricated_duration: fabricated_duration, &block)
     end
 
     control = observations.detect { |o| o.name == name }
@@ -290,4 +298,10 @@ def raise_on_mismatches?
       !!raise_on_mismatches
     end
   end
+
+  # Provide predefined durations to use instead of actual timing data.
+  # This is here solely as a convenience for developers of libraries that extend Scientist.
+  def fabricate_durations_for_testing_purposes(fabricated_durations = {})
+    @_scientist_fabricated_durations = fabricated_durations
+  end
 end

lib/scientist/observation.rb

@@ -23,19 +23,20 @@ class Scientist::Observation
   # The Float seconds elapsed.
   attr_reader :duration
 
-  def initialize(name, experiment, &block)
+  def initialize(name, experiment, fabricated_duration: nil, &block)
     @name       = name
     @experiment = experiment
     @now        = Time.now
 
-    starting = Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second)
+    starting = Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) unless fabricated_duration
     begin
       @value = block.call
     rescue *RESCUES => e
       @exception = e
     end
 
-    @duration = Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) - starting
+    @duration = fabricated_duration ||
+      Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) - starting
 
     freeze
   end

test/scientist/experiment_test.rb

@@ -239,6 +239,13 @@ def @ex.enabled?
     assert_equal 10, @ex.clean_value(10)
   end
 
+  it "provides the clean block when asked for it, in case subclasses wish to override and provide defaults" do
+    assert_nil @ex.cleaner
+    cleaner = ->(value) { value.upcase }
+    @ex.clean(&cleaner)
+    assert_equal cleaner, @ex.cleaner
+  end
+
   it "calls the configured clean block with a value when configured" do
     @ex.clean do |value|
       value.upcase
@@ -559,4 +566,32 @@ def @ex.enabled?
       refute before, "before_run should not have run"
     end
   end
+
+  describe "testing hooks for extending code" do
+    it "allows a user to provide fabricated durations for testing purposes" do
+      @ex.use { true }
+      @ex.try { true }
+      @ex.fabricate_durations_for_testing_purposes( "control" => 0.5, "candidate" => 1.0 )
+
+      @ex.run
+
+      cont = @ex.published_result.control
+      cand = @ex.published_result.candidates.first
+      assert_in_delta 0.5, cont.duration, 0.01
+      assert_in_delta 1.0, cand.duration, 0.01
+    end
+
+    it "returns actual durations if fabricated ones are omitted for some blocks" do
+      @ex.use { true }
+      @ex.try { sleep 0.1; true }
+      @ex.fabricate_durations_for_testing_purposes( "control" => 0.5 )
+
+      @ex.run
+
+      cont = @ex.published_result.control
+      cand = @ex.published_result.candidates.first
+      assert_in_delta 0.5, cont.duration, 0.01
+      assert_in_delta 0.1, cand.duration, 0.01
+    end
+  end
 end