This reverts commit 7acd239c70 to further
debug upstream issues.
This commit is contained in:
@@ -1,70 +0,0 @@
|
||||
# Backoffable provides transient-error retry backoff for models that depend on external services.
|
||||
#
|
||||
# When a transient error occurs (network hiccup, SSL failure, etc.) call apply_backoff!.
|
||||
# The wait time ramps from 1 minute up to BACKOFF_MAX_INTERVAL_MINUTES, then holds at that
|
||||
# ceiling for BACKOFF_MAX_INTERVAL_COUNT more attempts before calling prompt_reauthorization!.
|
||||
#
|
||||
# Call clear_backoff! after a successful operation to reset the counter.
|
||||
|
||||
module Backoffable
|
||||
extend ActiveSupport::Concern
|
||||
|
||||
def backoff_log_identifier
|
||||
inbox_id = respond_to?(:inbox) && inbox&.id
|
||||
inbox_id ? "#{self.class.name} - #{inbox_id}" : "#{self.class.name}##{id}"
|
||||
end
|
||||
|
||||
def backoff_retry_count
|
||||
::Redis::Alfred.get(backoff_retry_count_key).to_i
|
||||
end
|
||||
|
||||
def in_backoff?
|
||||
val = ::Redis::Alfred.get(backoff_retry_after_key)
|
||||
val.present? && Time.zone.at(val.to_f) > Time.current
|
||||
end
|
||||
|
||||
def apply_backoff!
|
||||
new_count = backoff_retry_count + 1
|
||||
max_interval, max_retries = backoff_limits
|
||||
|
||||
if new_count > max_retries
|
||||
exhaust_backoff(new_count)
|
||||
else
|
||||
schedule_backoff_retry(new_count, max_interval, max_retries)
|
||||
end
|
||||
end
|
||||
|
||||
def clear_backoff!
|
||||
::Redis::Alfred.delete(backoff_retry_count_key)
|
||||
::Redis::Alfred.delete(backoff_retry_after_key)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def backoff_limits
|
||||
max_interval = GlobalConfigService.load('BACKOFF_MAX_INTERVAL_MINUTES', 5).to_i
|
||||
max_count = GlobalConfigService.load('BACKOFF_MAX_INTERVAL_COUNT', 10).to_i
|
||||
[max_interval, (max_interval - 1) + max_count]
|
||||
end
|
||||
|
||||
def exhaust_backoff(new_count)
|
||||
Rails.logger.warn "#{backoff_log_identifier} backoff exhausted (#{new_count} failures), prompting reauthorization"
|
||||
clear_backoff!
|
||||
prompt_reauthorization!
|
||||
end
|
||||
|
||||
def schedule_backoff_retry(new_count, max_interval, max_retries)
|
||||
wait_minutes = [new_count, max_interval].min
|
||||
::Redis::Alfred.set(backoff_retry_count_key, new_count.to_s, ex: 24.hours)
|
||||
::Redis::Alfred.set(backoff_retry_after_key, wait_minutes.minutes.from_now.to_f.to_s, ex: 24.hours)
|
||||
Rails.logger.warn "#{backoff_log_identifier} backoff retry #{new_count}/#{max_retries}, next attempt in #{wait_minutes}m"
|
||||
end
|
||||
|
||||
def backoff_retry_count_key
|
||||
format(::Redis::Alfred::BACKOFF_RETRY_COUNT, obj_type: self.class.table_name.singularize, obj_id: id)
|
||||
end
|
||||
|
||||
def backoff_retry_after_key
|
||||
format(::Redis::Alfred::BACKOFF_RETRY_AFTER, obj_type: self.class.table_name.singularize, obj_id: id)
|
||||
end
|
||||
end
|
||||
@@ -13,8 +13,6 @@
|
||||
module Reauthorizable
|
||||
extend ActiveSupport::Concern
|
||||
|
||||
include Backoffable
|
||||
|
||||
AUTHORIZATION_ERROR_THRESHOLD = 2
|
||||
|
||||
# model attribute
|
||||
@@ -67,7 +65,6 @@ module Reauthorizable
|
||||
def reauthorized!
|
||||
::Redis::Alfred.delete(authorization_error_count_key)
|
||||
::Redis::Alfred.delete(reauthorization_required_key)
|
||||
clear_backoff!
|
||||
|
||||
invalidate_inbox_cache unless instance_of?(::AutomationRule)
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user