From 37f5435edd7c064156f5ccc9a50c5983f9ec8147 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Fri, 31 Mar 2023 05:38:47 -0700 Subject: [PATCH] IndexingScheduler: fetch and import in batches (#24285) Co-authored-by: Claire --- app/workers/scheduler/indexing_scheduler.rb | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/app/workers/scheduler/indexing_scheduler.rb b/app/workers/scheduler/indexing_scheduler.rb index f52d0141d4..09e7bd69e5 100644 --- a/app/workers/scheduler/indexing_scheduler.rb +++ b/app/workers/scheduler/indexing_scheduler.rb @@ -3,22 +3,20 @@ class Scheduler::IndexingScheduler include Sidekiq::Worker include Redisable - include DatabaseHelper - sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 30.minutes.to_i - - IMPORT_BATCH_SIZE = 1000 - SCAN_BATCH_SIZE = 10 * IMPORT_BATCH_SIZE + sidekiq_options retry: 0 def perform return unless Chewy.enabled? indexes.each do |type| with_redis do |redis| - redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE).each_slice(IMPORT_BATCH_SIZE) do |ids| - type.import!(ids) + ids = redis.smembers("chewy:queue:#{type.name}") - redis.srem("chewy:queue:#{type.name}", ids) + type.import!(ids) + + redis.pipelined do |pipeline| + ids.each { |id| pipeline.srem("chewy:queue:#{type.name}", id) } end end end @@ -27,6 +25,6 @@ class Scheduler::IndexingScheduler private def indexes - [AccountsIndex, TagsIndex, PublicStatusesIndex, StatusesIndex] + [AccountsIndex, TagsIndex, StatusesIndex] end end