From 2d06c3cefb9862e6f5b6e53f459e0e3a7cb52c6b Mon Sep 17 00:00:00 2001 From: Dirk Gadsden Date: Sun, 30 Apr 2017 21:24:51 -0700 Subject: [PATCH 1/7] Make Sidekiq worker options work with both `sidekiq-unique-jobs` and Sidekiq Enterprise --- lib/async_cache/workers/sidekiq.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/async_cache/workers/sidekiq.rb b/lib/async_cache/workers/sidekiq.rb index 8368e38..9b56793 100644 --- a/lib/async_cache/workers/sidekiq.rb +++ b/lib/async_cache/workers/sidekiq.rb @@ -7,8 +7,12 @@ class SidekiqWorker include Base include Sidekiq::Worker - # Only allow one job per set of arguments to ever be in the queue - sidekiq_options :unique => :until_executed + if defined?(Sidekiq::Enterprise) + sidekiq_options unique_for: 10.minutes + elsif defined?(SidekiqUniqueJobs) + # Only allow one job per set of arguments to ever be in the queue + sidekiq_options unique: :until_executed + end # Use the Sidekiq API to see if there are worker processes available to # handle the async cache jobs queue. From 378cf7788fbc4a339f88e69eeed8572db71b9a56 Mon Sep 17 00:00:00 2001 From: Dirk Gadsden Date: Sun, 30 Apr 2017 21:34:00 -0700 Subject: [PATCH 2/7] Update `json` gem to fix incompatbility with newer Rubies --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index bee00c6..18f5589 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -75,7 +75,7 @@ GEM json (~> 1.8) multi_xml (>= 0.5.2) i18n (0.7.0) - json (1.8.3) + json (1.8.6) loofah (2.0.3) nokogiri (>= 1.5.9) mail (2.6.3) @@ -204,4 +204,4 @@ DEPENDENCIES yard (~> 0.8) BUNDLED WITH - 1.11.2 + 1.14.6 From 96c6c28b9cb9c8ee0c4626be19e973945df6932f Mon Sep 17 00:00:00 2001 From: Dirk Gadsden Date: Sun, 30 Apr 2017 21:34:17 -0700 Subject: [PATCH 3/7] Add spec to ensure Sidekiq worker sets it options correctly --- spec/spec_helper.rb | 4 ++++ spec/workers/sidekiq_spec.rb | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index c152f09..f5d9948 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -11,4 +11,8 @@ Rails.cache = ActiveSupport::Cache::MemoryStore.new Rails.logger = Logger.new($stdout).tap { |log| log.level = Logger::ERROR } +# Fake the `sidekiq-unique-jobs` gem being loaded. +module SidekiqUniqueJobs +end + require 'async_cache' diff --git a/spec/workers/sidekiq_spec.rb b/spec/workers/sidekiq_spec.rb index 2575025..3b6fe39 100644 --- a/spec/workers/sidekiq_spec.rb +++ b/spec/workers/sidekiq_spec.rb @@ -6,6 +6,13 @@ AsyncCache::Workers::SidekiqWorker end + describe '.sidekiq_options' do + # See `spec_helper.rb` which makes it think the gem is loaded. + it 'has the uniqueness option for `sidekiq-unique-jobs`' do + expect(subject.sidekiq_options_hash).to include 'unique' + end + end + describe '::has_workers?' do it 'returns false if no Sidekiq queues are available' do allow(subject).to receive(:sidekiq_options).and_return({'queue' => 'good_queue'}) From 32d02df3f063a8ce3506097bde2fce9a7e31dfa0 Mon Sep 17 00:00:00 2001 From: Dirk Gadsden Date: Sun, 30 Apr 2017 21:42:11 -0700 Subject: [PATCH 4/7] Pull detection out into separate module and add specs for it --- lib/async_cache/workers/sidekiq.rb | 17 ++++++++++++----- spec/workers/sidekiq_spec.rb | 30 ++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/lib/async_cache/workers/sidekiq.rb b/lib/async_cache/workers/sidekiq.rb index 9b56793..a493964 100644 --- a/lib/async_cache/workers/sidekiq.rb +++ b/lib/async_cache/workers/sidekiq.rb @@ -7,13 +7,20 @@ class SidekiqWorker include Base include Sidekiq::Worker - if defined?(Sidekiq::Enterprise) - sidekiq_options unique_for: 10.minutes - elsif defined?(SidekiqUniqueJobs) - # Only allow one job per set of arguments to ever be in the queue - sidekiq_options unique: :until_executed + # Pulled out into a module so it can be tested. + module Options + def self.included(mod) + if defined?(Sidekiq::Enterprise) + mod.sidekiq_options unique_for: 10.minutes + elsif defined?(SidekiqUniqueJobs) + # Only allow one job per set of arguments to ever be in the queue + mod.sidekiq_options unique: :until_executed + end + end end + include Options + # Use the Sidekiq API to see if there are worker processes available to # handle the async cache jobs queue. def self.has_workers? diff --git a/spec/workers/sidekiq_spec.rb b/spec/workers/sidekiq_spec.rb index 3b6fe39..f0471fe 100644 --- a/spec/workers/sidekiq_spec.rb +++ b/spec/workers/sidekiq_spec.rb @@ -44,4 +44,34 @@ ) end end + + describe AsyncCache::Workers::SidekiqWorker::Options do + subject do + AsyncCache::Workers::SidekiqWorker::Options + end + + before do + # Set by `spec_helper.rb`. + hide_const 'SidekiqUniqueJobs' + + class Worker + end + end + + it 'sets correct option for `sidekiq-unique-jobs`' do + stub_const 'SidekiqUniqueJobs', Module.new + + expect(Worker).to receive(:sidekiq_options).with(unique: :until_executed) + + Worker.include subject + end + + it 'sets correct options for Sidekiq Enterprise' do + stub_const 'Sidekiq::Enterprise', Module.new + + expect(Worker).to receive(:sidekiq_options).with(unique_for: 10.minutes) + + Worker.include subject + end + end end From 4b5cb6f24fe30da76b4c756593a896b4b5492f1c Mon Sep 17 00:00:00 2001 From: Dirk Gadsden Date: Mon, 21 May 2018 15:12:05 -0700 Subject: [PATCH 5/7] Add `AsyncCache.options` in the same pattern as `Sidekiq.options` --- lib/async_cache.rb | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/lib/async_cache.rb b/lib/async_cache.rb index 9440d66..ab3fc4a 100644 --- a/lib/async_cache.rb +++ b/lib/async_cache.rb @@ -1,20 +1,30 @@ require 'sourcify' module AsyncCache - class << self - def backend - @backend || Rails.cache - end - def backend=(backend) - @backend = backend - end + DEFAULT_OPTIONS = { + # How long Sidekiq Enterprise should hold a uniqueness lock. + uniqueness_timeout: 10.minutes, + } - def logger - @logger || Rails.logger - end - def logger=(logger) - @logger = logger - end + def self.options + @options ||= DEFAULT_OPTIONS.dup + end + def self.options=(options) + @options = options + end + + def self.backend + @backend ||= Rails.cache + end + def self.backend=(backend) + @backend = backend + end + + def self.logger + @logger ||= Rails.logger + end + def self.logger=(logger) + @logger = logger end end From f9778a499bfb99dfc4f734fd5ff5ea046b8119fb Mon Sep 17 00:00:00 2001 From: Dirk Gadsden Date: Mon, 21 May 2018 15:12:22 -0700 Subject: [PATCH 6/7] Make worker read uniqueness timeout from `AsyncCache.options` --- lib/async_cache/workers/sidekiq.rb | 2 +- spec/workers/sidekiq_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/async_cache/workers/sidekiq.rb b/lib/async_cache/workers/sidekiq.rb index a493964..36994d6 100644 --- a/lib/async_cache/workers/sidekiq.rb +++ b/lib/async_cache/workers/sidekiq.rb @@ -11,7 +11,7 @@ class SidekiqWorker module Options def self.included(mod) if defined?(Sidekiq::Enterprise) - mod.sidekiq_options unique_for: 10.minutes + mod.sidekiq_options unique_for: AsyncCache.options[:uniqueness_timeout] elsif defined?(SidekiqUniqueJobs) # Only allow one job per set of arguments to ever be in the queue mod.sidekiq_options unique: :until_executed diff --git a/spec/workers/sidekiq_spec.rb b/spec/workers/sidekiq_spec.rb index f0471fe..cb1702f 100644 --- a/spec/workers/sidekiq_spec.rb +++ b/spec/workers/sidekiq_spec.rb @@ -69,7 +69,7 @@ class Worker it 'sets correct options for Sidekiq Enterprise' do stub_const 'Sidekiq::Enterprise', Module.new - expect(Worker).to receive(:sidekiq_options).with(unique_for: 10.minutes) + expect(Worker).to receive(:sidekiq_options).with(unique_for: AsyncCache.options[:uniqueness_timeout]) Worker.include subject end From 0bba991708e4c258cf2ad0f4d5deed9dc0470490 Mon Sep 17 00:00:00 2001 From: Dirk Gadsden Date: Sun, 17 Jun 2018 17:16:14 -0700 Subject: [PATCH 7/7] Don't assume `active_support/core_ext` is available --- lib/async_cache.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/async_cache.rb b/lib/async_cache.rb index ab3fc4a..664b3a6 100644 --- a/lib/async_cache.rb +++ b/lib/async_cache.rb @@ -2,8 +2,9 @@ module AsyncCache DEFAULT_OPTIONS = { - # How long Sidekiq Enterprise should hold a uniqueness lock. - uniqueness_timeout: 10.minutes, + # How long Sidekiq Enterprise should hold a uniqueness lock. The default + # is 10 minutes. + uniqueness_timeout: 600, } def self.options