Wait for a random interval before starting the scheduler's tasks

This comes from an idea from @djmb, so that, when we have multiple schedulers,
they don't start all at the same time after a deploy, enqueuing All The Things
and causing a thundering herd.

Author: rosa

Diff for: lib/solid_queue/interruptible.rb

@@ -17,7 +17,7 @@ def interrupt
     end
 
     def interruptible_sleep(time)
-      if self_pipe[:reader].wait_readable(time)
+      if time > 0 && self_pipe[:reader].wait_readable(time)
         loop { self_pipe[:reader].read_nonblock(SELF_PIPE_BLOCK_SIZE) }
       end
     rescue Errno::EAGAIN, Errno::EINTR

Diff for: lib/solid_queue/runner.rb

@@ -11,6 +11,7 @@ module Runner
 
     def start(mode: :supervised)
       boot_in mode
+      observe_starting_delay
 
       run_callbacks(:start) do
         if mode == :async
@@ -44,6 +45,10 @@ def boot_in(mode)
         SolidQueue.logger.info("[SolidQueue] Starting #{self}")
       end
 
+      def observe_starting_delay
+        interruptible_sleep(initial_jitter)
+      end
+
       def register_signal_handlers
         %w[ INT TERM ].each do |signal|
           trap(signal) do
@@ -81,6 +86,10 @@ def shutdown
         procline "shutting down"
       end
 
+      def initial_jitter
+        0
+      end
+
       def stopping?
         @stopping
       end

Diff for: lib/solid_queue/scheduler.rb

@@ -58,6 +58,10 @@ def unblock_blocked_executions
         BlockedExecution.unblock(batch_size)
       end
 
+      def initial_jitter
+        Kernel.rand(0...polling_interval)
+      end
+
       def metadata
         super.merge(batch_size: batch_size, polling_interval: polling_interval)
       end