More realistic row overhead estimate

Some experimentation suggests the row overhead for MySQL and Postgresql
is 130-150 bytes beyond the size of the key and value columns.

On Sqlite it is more like 90 bytes though, but lets go for the worst
case.

Author: djmb

app/models/solid_cache/entry.rb

@@ -4,12 +4,10 @@ module SolidCache
   class Entry < Record
     include Expiration, Size
 
-    ID_BYTE_SIZE = 8
-    CREATED_AT_BYTE_SIZE = 8
-    KEY_HASH_BYTE_SIZE = 8
-    VALUE_BYTE_SIZE = 4
-    FIXED_SIZE_COLUMNS_BYTE_SIZE = ID_BYTE_SIZE + CREATED_AT_BYTE_SIZE + KEY_HASH_BYTE_SIZE + VALUE_BYTE_SIZE
-
+    # The estimated cost of an extra row in bytes, including fixed size columns, overhead, indexes and free space
+    # Based on expirimentation on SQLite, MySQL and Postgresql.
+    # A bit high for SQLite (more like 90 bytes), but about right for MySQL/Postgresql.
+    ESTIMATED_ROW_OVERHEAD = 140
     KEY_HASH_ID_RANGE = -(2**63)..(2**63 - 1)
 
     class << self
@@ -167,7 +165,7 @@ def key_hash_for(key)
         end
 
         def byte_size_for(payload)
-          payload[:key].to_s.bytesize + payload[:value].to_s.bytesize + FIXED_SIZE_COLUMNS_BYTE_SIZE
+          payload[:key].to_s.bytesize + payload[:value].to_s.bytesize + ESTIMATED_ROW_OVERHEAD
         end
     end
   end

docker-compose.yml

@@ -1,21 +1,22 @@
 version: "3.8"
 
 volumes:
-  db: {}
+  mysql: {}
+  postgres: {}
 
 services:
   postgres:
     image: postgres:15.1
     environment:
       POSTGRES_HOST_AUTH_METHOD: "trust"
     volumes:
-      - db:/var/lib/postgres
+      - postgres:/var/lib/postgres
     ports: [ "127.0.0.1:55432:5432" ]
   mysql:
     image: mysql:8.0.31
     environment:
       MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
     volumes:
-      - db:/var/lib/mysql
+      - mysql:/var/lib/mysql
     ports: [ "127.0.0.1:33060:3306" ]
 

test/dummy/config/environments/development.rb

@@ -30,7 +30,7 @@
   else
     config.action_controller.perform_caching = false
 
-    config.cache_store = :null_store
+    config.cache_store = :solid_cache_store
   end
 
   # Print deprecation notices to the Rails logger.

test/models/solid_cache/entry/size/estimate_test.rb

@@ -11,21 +11,21 @@ class EntrySizeEstimateTest < ActiveSupport::TestCase
     test "gets exact estimate when samples sizes are big enough" do
       write_entries(value_lengths: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ])
 
-      assert_equal 415, estimate(samples: 12)
-      assert_equal 415, estimate(samples: 10)
-      assert_equal 456, estimate(samples: 6)
-      assert_equal 457, estimate(samples: 5)
+      assert_equal 1535, estimate(samples: 12)
+      assert_equal 1535, estimate(samples: 10)
+      assert_equal 1688, estimate(samples: 6)
+      assert_equal 1689, estimate(samples: 5)
     end
 
     test "test larger sample estimates" do
       values_lengths = with_fixed_srand(1) { 1000.times.map { (rand**2 * 1000).to_i } }
       write_entries(value_lengths: values_lengths)
 
-      assert_equal 369257, estimate(samples: 1000)
-      assert_equal 369550, estimate(samples: 500)
-      with_fixed_srand(1) { assert_equal 383576, estimate(samples: 100) }
-      with_fixed_srand(1) { assert_equal 357109, estimate(samples: 50) }
-      with_fixed_srand(1) { assert_equal 326614, estimate(samples: 10) }
+      assert_equal 481257, estimate(samples: 1000)
+      assert_equal 481662, estimate(samples: 500)
+      with_fixed_srand(1) { assert_equal 501624, estimate(samples: 100) }
+      with_fixed_srand(1) { assert_equal 477621, estimate(samples: 50) }
+      with_fixed_srand(1) { assert_equal 471878, estimate(samples: 10) }
     end
 
     test "test with gaps in records estimates" do
@@ -34,12 +34,12 @@ class EntrySizeEstimateTest < ActiveSupport::TestCase
       first_mod = Entry.first.id % 3
       Entry.where("id % 3 = #{first_mod}").delete_all
 
-      assert_equal 249940, estimate(samples: 1000)
-      assert_equal 250037, estimate(samples: 500)
-      with_fixed_srand(1) { assert_equal 249354, estimate(samples: 334) }
-      with_fixed_srand(1) { assert_equal 267523, estimate(samples: 100) }
-      with_fixed_srand(1) { assert_equal 257970, estimate(samples: 50) }
-      with_fixed_srand(1) { assert_equal 203365, estimate(samples: 10) }
+      assert_equal 324532, estimate(samples: 1000)
+      assert_equal 324741, estimate(samples: 500)
+      with_fixed_srand(1) { assert_equal 323946, estimate(samples: 334) }
+      with_fixed_srand(1) { assert_equal 345103, estimate(samples: 100) }
+      with_fixed_srand(1) { assert_equal 335770, estimate(samples: 50) }
+      with_fixed_srand(1) { assert_equal 281944, estimate(samples: 10) }
     end
 
     test "test with more gaps in records estimates" do
@@ -48,12 +48,12 @@ class EntrySizeEstimateTest < ActiveSupport::TestCase
       first_mod = Entry.first.id % 4
       Entry.where("id % 4 != #{first_mod}").delete_all
 
-      assert_equal 92304, estimate(samples: 1000)
-      assert_equal 92592, estimate(samples: 500)
-      with_fixed_srand(1) { assert_equal 92519, estimate(samples: 250) }
-      with_fixed_srand(1) { assert_equal 95475, estimate(samples: 100) }
-      with_fixed_srand(1) { assert_equal 101601, estimate(samples: 50) }
-      with_fixed_srand(1) { assert_equal 13362, estimate(samples: 10) }
+      assert_equal 120304, estimate(samples: 1000)
+      assert_equal 121488, estimate(samples: 500)
+      with_fixed_srand(1) { assert_equal 121188, estimate(samples: 250) }
+      with_fixed_srand(1) { assert_equal 126768, estimate(samples: 100) }
+      with_fixed_srand(1) { assert_equal 132657, estimate(samples: 50) }
+      with_fixed_srand(1) { assert_equal 25537, estimate(samples: 10) }
     end
 
     test "overestimate when all samples sizes are the same" do
@@ -62,11 +62,11 @@ class EntrySizeEstimateTest < ActiveSupport::TestCase
       # estimate in this case.
       write_entries(value_lengths: [1] * 1000)
 
-      assert_equal 37000, estimate(samples: 1000)
-      assert_equal 73963, estimate(samples: 999)
-      assert_equal 55500, estimate(samples: 500)
-      with_fixed_srand(1) { assert_equal 67648, estimate(samples: 6) }
-      with_fixed_srand(1) { assert_equal 81178, estimate(samples: 5) }
+      assert_equal 149000, estimate(samples: 1000)
+      assert_equal 297851, estimate(samples: 999)
+      assert_equal 223500, estimate(samples: 500)
+      with_fixed_srand(1) { assert_equal 272422, estimate(samples: 6) }
+      with_fixed_srand(1) { assert_equal 326906, estimate(samples: 5) }
     end
 
     private

test/models/solid_cache/entry/size/moving_average_estimate_test.rb

@@ -13,7 +13,7 @@ class EntrySizeMovingAverageEstimateTest < ActiveSupport::TestCase
 
       estimate = Entry::Size::MovingAverageEstimate.new(samples: 12)
       assert_predicate estimate, :exact?
-      assert_equal 415, estimate.size
+      assert_equal 1535, estimate.size
     end
 
     test "tracks moving average" do
@@ -22,24 +22,24 @@ class EntrySizeMovingAverageEstimateTest < ActiveSupport::TestCase
       Entry.write Entry::Size::MovingAverageEstimate::ESTIMATES_KEY, "4637774|4754378|7588547"
 
       with_fixed_srand(1) do
-        assert_equal 10075987, estimate(samples: 1)
+        assert_equal 10449357, estimate(samples: 1)
       end
 
-      assert_equal "4754378|7588547|17885035", Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
+      assert_equal "4754378|7588547|19005147", Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
     end
 
     test "appends to moving average when less than required items" do
       write_entries(value_lengths: 5000.times.to_a)
 
       assert_nil Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
 
-      with_fixed_srand(1) { assert_equal 19872121, estimate(samples: 2) }
+      with_fixed_srand(1) { assert_equal 20991897, estimate(samples: 2) }
 
-      assert_equal "19872121", Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
+      assert_equal "20991897", Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
 
-      with_fixed_srand(2) { assert_equal 11077118, estimate(samples: 2) }
+      with_fixed_srand(2) { assert_equal 11917062, estimate(samples: 2) }
 
-      assert_equal "19872121|2282115", Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
+      assert_equal "20991897|2842227", Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
     end
 
     private

test/models/solid_cache/entry/size_test.rb

@@ -11,20 +11,20 @@ class EntrySizeTest < ActiveSupport::TestCase
     test "gets exact estimate when samples sizes are big enough" do
       write_entries(value_lengths: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ])
 
-      assert_equal 415, Entry.estimated_size(samples: 12)
-      assert_equal 533, Entry.estimated_size(samples: 10)
-      assert_equal 537, Entry.estimated_size(samples: 6)
+      assert_equal 1535, Entry.estimated_size(samples: 12)
+      assert_equal 1878, Entry.estimated_size(samples: 10)
+      assert_equal 1882, Entry.estimated_size(samples: 6)
     end
 
     test "test larger sample estimates" do
       values_lengths = with_fixed_srand(1) { 1000.times.map { (rand**2 * 1000).to_i } }
       write_entries(value_lengths: values_lengths)
 
-      assert_equal 369257, Entry.estimated_size(samples: 1000)
-      assert_equal 369926, Entry.estimated_size(samples: 501)
-      with_fixed_srand(1) { assert_equal 383898, Entry.estimated_size(samples: 100) }
-      with_fixed_srand(1) { assert_equal 357433, Entry.estimated_size(samples: 50) }
-      with_fixed_srand(1) { assert_equal 326934, Entry.estimated_size(samples: 10) }
+      assert_equal 481257, Entry.estimated_size(samples: 1000)
+      assert_equal 482262, Entry.estimated_size(samples: 501)
+      with_fixed_srand(1) { assert_equal 502065, Entry.estimated_size(samples: 100) }
+      with_fixed_srand(1) { assert_equal 478066, Entry.estimated_size(samples: 50) }
+      with_fixed_srand(1) { assert_equal 472343, Entry.estimated_size(samples: 10) }
     end
 
     test "test with gaps in records estimates" do
@@ -33,12 +33,12 @@ class EntrySizeTest < ActiveSupport::TestCase
       first_mod = Entry.first.id % 3
       Entry.where("id % 3 = #{first_mod}").delete_all
 
-      assert_equal 249940, Entry.estimated_size(samples: 1000)
-      assert_equal 250120, Entry.estimated_size(samples: 500)
-      with_fixed_srand(1) { assert_equal 249639, Entry.estimated_size(samples: 334) }
-      with_fixed_srand(1) { assert_equal 267921, Entry.estimated_size(samples: 100) }
-      with_fixed_srand(1) { assert_equal 258414, Entry.estimated_size(samples: 50) }
-      with_fixed_srand(1) { assert_equal 203756, Entry.estimated_size(samples: 10) }
+      assert_equal 324532, Entry.estimated_size(samples: 1000)
+      assert_equal 324936, Entry.estimated_size(samples: 500)
+      with_fixed_srand(1) { assert_equal 324567, Entry.estimated_size(samples: 334) }
+      with_fixed_srand(1) { assert_equal 345649, Entry.estimated_size(samples: 100) }
+      with_fixed_srand(1) { assert_equal 336366, Entry.estimated_size(samples: 50) }
+      with_fixed_srand(1) { assert_equal 282492, Entry.estimated_size(samples: 10) }
     end
 
     test "test with more gaps in records estimates" do
@@ -47,12 +47,12 @@ class EntrySizeTest < ActiveSupport::TestCase
       first_mod = Entry.first.id % 4
       Entry.where("id % 4 != #{first_mod}").delete_all
 
-      assert_equal 92304, Entry.estimated_size(samples: 1000)
-      assert_equal 92674, Entry.estimated_size(samples: 501)
-      with_fixed_srand(1) { assert_equal 92566, Entry.estimated_size(samples: 250) }
-      with_fixed_srand(1) { assert_equal 95594, Entry.estimated_size(samples: 100) }
-      with_fixed_srand(1) { assert_equal 101852, Entry.estimated_size(samples: 50) }
-      with_fixed_srand(1) { assert_equal 13377, Entry.estimated_size(samples: 10) }
+      assert_equal 120304, Entry.estimated_size(samples: 1000)
+      assert_equal 121683, Entry.estimated_size(samples: 501)
+      with_fixed_srand(1) { assert_equal 121240, Entry.estimated_size(samples: 250) }
+      with_fixed_srand(1) { assert_equal 126976, Entry.estimated_size(samples: 100) }
+      with_fixed_srand(1) { assert_equal 133014, Entry.estimated_size(samples: 50) }
+      with_fixed_srand(1) { assert_equal 25596, Entry.estimated_size(samples: 10) }
     end
 
     test "overestimate when all samples sizes are the same" do
@@ -61,11 +61,11 @@ class EntrySizeTest < ActiveSupport::TestCase
       # estimate in this case.
       write_entries(value_lengths: [1] * 1000)
 
-      assert_equal 37000, Entry.estimated_size(samples: 1000)
-      assert_equal 74008, Entry.estimated_size(samples: 999)
-      assert_equal 55582, Entry.estimated_size(samples: 501)
-      with_fixed_srand(1) { assert_equal 67761, Entry.estimated_size(samples: 6) }
-      with_fixed_srand(1) { assert_equal 81304, Entry.estimated_size(samples: 5) }
+      assert_equal 149000, Entry.estimated_size(samples: 1000)
+      assert_equal 297897, Entry.estimated_size(samples: 999)
+      assert_equal 223695, Entry.estimated_size(samples: 501)
+      with_fixed_srand(1) { assert_equal 272741, Entry.estimated_size(samples: 6) }
+      with_fixed_srand(1) { assert_equal 327280, Entry.estimated_size(samples: 5) }
     end
 
     private

test/models/solid_cache/entry_test.rb

@@ -54,11 +54,11 @@ class EntryTest < ActiveSupport::TestCase
 
     test "byte_size" do
       Entry.write "hello".b, "test"
-      assert_equal 37, Entry.uncached { Entry.last.byte_size }
+      assert_equal 149, Entry.uncached { Entry.last.byte_size }
       Entry.write "hello".b, "12345"
-      assert_equal 38, Entry.uncached { Entry.last.byte_size }
+      assert_equal 150, Entry.uncached { Entry.last.byte_size }
       Entry.write "hi".b, "12345"
-      assert_equal 35, Entry.uncached { Entry.last.byte_size }
+      assert_equal 147, Entry.uncached { Entry.last.byte_size }
     end
 
     private