Implement Sidekiq for background jobs and update rate limiting

- Add DeleteExpiredPayloadsJob for automatic payload expiration
- Integrate Sidekiq for background job processing
- Update rate limiting implementation in RateLimitable concern
- Add tests for rate limiting and background jobs
- Update documentation (README.md, kamal.md, progress.md) with Sidekiq details
- Include instructions for running and deploying with Sidekiq

Author: codevalley

Gemfile

@@ -54,5 +54,9 @@ gem "importmap-rails"
 group :test do
   gem "simplecov", require: false
   gem "simplecov-cobertura"
-  gem "fakeredis"
+  gem "fakeredis", "~> 0.8.0"
 end
+
+# Add these lines
+gem "sidekiq"
+gem "sidekiq-scheduler"

Gemfile.lock

@@ -102,7 +102,8 @@ GEM
     factory_bot_rails (6.4.3)
       factory_bot (~> 6.4)
       railties (>= 5.0.0)
-    fakeredis (0.1.4)
+    fakeredis (0.8.0)
+      redis (~> 4.1)
     fugit (1.11.1)
       et-orbi (~> 1, >= 1.2.11)
       raabro (~> 1.4)
@@ -198,8 +199,7 @@ GEM
     rake (13.2.1)
     rdoc (6.7.0)
       psych (>= 4.0.0)
-    redis (5.3.0)
-      redis-client (>= 0.22.0)
+    redis (4.8.1)
     redis-client (0.22.2)
       connection_pool
     regexp_parser (2.9.2)
@@ -259,6 +259,16 @@ GEM
     securerandom (0.3.1)
     shoulda-matchers (6.4.0)
       activesupport (>= 5.2.0)
+    sidekiq (7.3.1)
+      concurrent-ruby (< 2)
+      connection_pool (>= 2.3.0)
+      logger
+      rack (>= 2.2.4)
+      redis-client (>= 0.22.2)
+    sidekiq-scheduler (5.0.6)
+      rufus-scheduler (~> 3.2)
+      sidekiq (>= 6, < 8)
+      tilt (>= 1.4.0, < 3)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -280,6 +290,7 @@ GEM
     stringio (3.1.1)
     strscan (3.1.0)
     thor (1.3.1)
+    tilt (2.4.0)
     timeout (0.4.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
@@ -304,7 +315,7 @@ DEPENDENCIES
   debug
   dotenv-rails
   factory_bot_rails
-  fakeredis
+  fakeredis (~> 0.8.0)
   importmap-rails
   puma
   rails (~> 7.2.1)
@@ -314,6 +325,8 @@ DEPENDENCIES
   rubocop-rails-omakase
   rufus-scheduler
   shoulda-matchers
+  sidekiq
+  sidekiq-scheduler
   simplecov
   simplecov-cobertura
   sprockets-rails

README.md

@@ -58,6 +58,13 @@ bundle exec rspec # check test coverage
 bundle exec rails test # check unit test coverage
 rails server # to locally start the server
 ```
+## Running service
+For local run you can use below command
+```bash
+rails server # to start the server
+bundle exec sidekiq # to start sidekiq
+```
+
 ## Deployment
 
 This project uses Kamal for deployment. Follow these steps to deploy:

app/controllers/concerns/rate_limitable.rb

@@ -8,19 +8,22 @@ module RateLimitable
   private
 
   def check_rate_limit
-    key = "rate_limit:#{request.ip}:#{controller_name}:#{action_name}"
-    count = REDIS.get(key).to_i
+    current_count = REDIS.get(rate_limit_key).to_i
 
-    if count >= rate_limit
-      render json: { error: "Rate limit exceeded" }, status: :too_many_requests
-    else
-      REDIS.multi do
-        REDIS.incr(key)
-        REDIS.expire(key, 1.hour.to_i)
-      end
+    REDIS.multi do |pipeline|
+      pipeline.incr(rate_limit_key)
+      pipeline.expire(rate_limit_key, 1.hour.to_i)
+    end
+
+    if current_count >= rate_limit
+      render json: { error: "Rate limit exceeded. Try again later." }, status: :too_many_requests
     end
   end
 
+  def rate_limit_key
+    "#{controller_name}:#{action_name}:#{request.ip}"
+  end
+
   def rate_limit
     case action_name.to_sym
     when :create

app/jobs/delete_expired_payloads_job.rb

@@ -0,0 +1,7 @@
+class DeleteExpiredPayloadsJob < ApplicationJob
+  queue_as :default
+
+  def perform
+    Payload.where("expiry_time <= ?", Time.current).destroy_all
+  end
+end

config/application.rb

@@ -35,5 +35,8 @@ class Application < Rails::Application
     config.assets.compile = false
     config.assets.precompile += %w[ manifest.js ]
     config.public_file_server.enabled = true
+
+    # Use Sidekiq as the job queue adapter
+    config.active_job.queue_adapter = :sidekiq
   end
 end

config/initializers/sidekiq.rb

@@ -0,0 +1,26 @@
+require "sidekiq"
+require "sidekiq-scheduler"
+
+redis_config = if Rails.env.test?
+                 { url: "redis://localhost:6379/0" }
+else
+                 { url: ENV["REDIS_URL"] || "redis://localhost:6379/0" }
+end
+
+Sidekiq.configure_server do |config|
+  config.redis = redis_config
+
+  if Rails.env.production?
+    config.on(:startup) do
+      schedule_file = File.expand_path("../../sidekiq.yml", __FILE__)
+      if File.exist?(schedule_file)
+        Sidekiq.schedule = YAML.load_file(schedule_file)
+        SidekiqScheduler::Scheduler.instance.reload_schedule!
+      end
+    end
+  end
+end
+
+Sidekiq.configure_client do |config|
+  config.redis = redis_config
+end

config/routes.rb

@@ -1,3 +1,6 @@
+require "sidekiq/web"
+require "sidekiq-scheduler/web"
+
 Rails.application.routes.draw do
   # Define your application routes per the DSL in https://guides.rubyonrails.org/routing.html
 
@@ -15,4 +18,7 @@
   end
 
   resources :payloads, only: [ :index, :show, :create, :update ]
+
+  # Mount Sidekiq web interface
+  mount Sidekiq::Web => "/sidekiq"
 end

config/sidekiq.yml

@@ -0,0 +1,4 @@
+:schedule:
+  delete_expired_payloads:
+    cron: '0 * * * *'  # Runs every hour
+    class: DeleteExpiredPayloadsJob

docker-compose.yml

@@ -9,7 +9,18 @@ services:
       - REDIS_URL=redis://redis:6379/0
     depends_on:
       - redis
+  sidekiq:
+    build: .
+    command: bundle exec sidekiq
+    volumes:
+      - .:/app
+    depends_on:
+      - redis
+    env_file:
+      - .env
   redis:
     image: redis:7-alpine
-    ports:
-      - "6379:6379"
+    volumes:
+      - redis:/data
+volumes:
+  redis:

public/assets/.sprockets-manifest-7c9b14749c7823575d1042837ce2a479.json

@@ -0,0 +1 @@
+{"files":{"manifest-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.js":{"logical_path":"manifest.js","mtime":"2024-08-30T15:01:58+05:30","size":0,"digest":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","integrity":"sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="},"actiontext-338257afa53cc53f94018abf7a897120d1eb3ee71f39e4641b2058442c9c4dc6.js":{"logical_path":"actiontext.js","mtime":"2024-08-30T15:01:58+05:30","size":30336,"digest":"338257afa53cc53f94018abf7a897120d1eb3ee71f39e4641b2058442c9c4dc6","integrity":"sha256-M4JXr6U8xT+UAYq/eolxINHrPucfOeRkGyBYRCycTcY="},"actiontext.esm-e787c6f528f0a88ee04de7c5ce4e3ab37fcf2691629ed21f0b30ee2db848c8c8.js":{"logical_path":"actiontext.esm.js","mtime":"2024-08-30T15:01:58+05:30","size":28519,"digest":"e787c6f528f0a88ee04de7c5ce4e3ab37fcf2691629ed21f0b30ee2db848c8c8","integrity":"sha256-54fG9SjwqI7gTefFzk46s3/PJpFintIfCzDuLbhIyMg="},"trix-fc94199b9d6e67e0780ad852719d0527834679c98a039ee492dedaae29f902df.js":{"logical_path":"trix.js","mtime":"2024-08-30T15:01:58+05:30","size":439967,"digest":"fc94199b9d6e67e0780ad852719d0527834679c98a039ee492dedaae29f902df","integrity":"sha256-/JQZm51uZ+B4CthScZ0FJ4NGecmKA57kkt7arin5At8="},"trix-4c7f56767699ae6c08f21004c9dc23a1b76b0734b0fca61d4534900ee0fef211.css":{"logical_path":"trix.css","mtime":"2024-08-30T15:01:58+05:30","size":20027,"digest":"4c7f56767699ae6c08f21004c9dc23a1b76b0734b0fca61d4534900ee0fef211","integrity":"sha256-TH9WdnaZrmwI8hAEydwjobdrBzSw/KYdRTSQDuD+8hE="},"activestorage-7dbadea816242c6d03cc2f59c58398ab55bb9a70ee859fa962ad4c7ad4c48594.js":{"logical_path":"activestorage.js","mtime":"2024-08-30T15:01:58+05:30","size":29379,"digest":"7dbadea816242c6d03cc2f59c58398ab55bb9a70ee859fa962ad4c7ad4c48594","integrity":"sha256-fbreqBYkLG0DzC9ZxYOYq1W7mnDuhZ+pYq1MetTEhZQ="},"activestorage.esm-b2d9aa45888a540d7e6969e484dbdb0a0a501ff56542ce737db6599b311e3385.js":{"logical_path":"activestorage.esm.js","mtime":"2024-08-30T15:01:58+05:30","size":27247,"digest":"b2d9aa45888a540d7e6969e484dbdb0a0a501ff56542ce737db6599b311e3385","integrity":"sha256-stmqRYiKVA1+aWnkhNvbCgpQH/VlQs5zfbZZmzEeM4U="},"actioncable-1a239ff96644bc8fd626f87a74175b4ee7eafd201f5034034a433c22d8c4dc3e.js":{"logical_path":"actioncable.js","mtime":"2024-08-30T15:01:58+05:30","size":16474,"digest":"1a239ff96644bc8fd626f87a74175b4ee7eafd201f5034034a433c22d8c4dc3e","integrity":"sha256-GiOf+WZEvI/WJvh6dBdbTufq/SAfUDQDSkM8ItjE3D4="},"actioncable.esm-555679e44f119be7a7e54121fca0beb0a5832f2933c1a1edcd0515bbafad22e0.js":{"logical_path":"actioncable.esm.js","mtime":"2024-08-30T15:01:58+05:30","size":14813,"digest":"555679e44f119be7a7e54121fca0beb0a5832f2933c1a1edcd0515bbafad22e0","integrity":"sha256-VVZ55E8Rm+en5UEh/KC+sKWDLykzwaHtzQUVu6+tIuA="}},"assets":{"manifest.js":"manifest-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.js","actiontext.js":"actiontext-338257afa53cc53f94018abf7a897120d1eb3ee71f39e4641b2058442c9c4dc6.js","actiontext.esm.js":"actiontext.esm-e787c6f528f0a88ee04de7c5ce4e3ab37fcf2691629ed21f0b30ee2db848c8c8.js","trix.js":"trix-fc94199b9d6e67e0780ad852719d0527834679c98a039ee492dedaae29f902df.js","trix.css":"trix-4c7f56767699ae6c08f21004c9dc23a1b76b0734b0fca61d4534900ee0fef211.css","activestorage.js":"activestorage-7dbadea816242c6d03cc2f59c58398ab55bb9a70ee859fa962ad4c7ad4c48594.js","activestorage.esm.js":"activestorage.esm-b2d9aa45888a540d7e6969e484dbdb0a0a501ff56542ce737db6599b311e3385.js","actioncable.js":"actioncable-1a239ff96644bc8fd626f87a74175b4ee7eafd201f5034034a433c22d8c4dc3e.js","actioncable.esm.js":"actioncable.esm-555679e44f119be7a7e54121fca0beb0a5832f2933c1a1edcd0515bbafad22e0.js"}}