diff --git a/lib/active_record/connection_adapters/postgresql_fallback_adapter.rb b/lib/active_record/connection_adapters/postgresql_fallback_adapter.rb index de37e131def..04b9840ccca 100644 --- a/lib/active_record/connection_adapters/postgresql_fallback_adapter.rb +++ b/lib/active_record/connection_adapters/postgresql_fallback_adapter.rb @@ -6,9 +6,10 @@ require 'sidekiq/pausable' class PostgreSQLFallbackHandler include Singleton - @@masters_down = DistributedCache.new('masters_down') + attr_reader :masters_down def initialize + @masters_down = DistributedCache.new('masters_down') @mutex = Mutex.new end @@ -20,38 +21,43 @@ class PostgreSQLFallbackHandler begin thread = Thread.new { initiate_fallback_to_master } thread.join - break if synchronize { @@masters_down.empty? } + break if synchronize { @masters_down.hash.empty? } sleep 10 ensure thread.kill end end end + + @thread.abort_on_exception = true end def master_down? - synchronize { @@masters_down[namespace] } + synchronize { @masters_down[namespace] } end def master_down=(args) synchronize do - @@masters_down[namespace] = args - Sidekiq.pause! if args + @masters_down[namespace] = args + Sidekiq.pause! if args && !Sidekiq.paused? end end def master_up(namespace) - synchronize { @@masters_down.delete(namespace) } + synchronize { @masters_down.delete(namespace) } end def initiate_fallback_to_master - @@masters_down.keys.each do |key| + @masters_down.hash.keys.each do |key| RailsMultisite::ConnectionManagement.with_connection(key) do begin logger.warn "#{log_prefix}: Checking master server..." - connection = ActiveRecord::Base.postgresql_connection(config) - is_connection_active = connection.active? - connection.disconnect! + begin + connection = ActiveRecord::Base.postgresql_connection(config) + is_connection_active = connection.active? + ensure + connection.disconnect! + end if is_connection_active logger.warn "#{log_prefix}: Master server is active. Reconnecting..." @@ -70,7 +76,7 @@ class PostgreSQLFallbackHandler # Use for testing def setup! - @@masters_down = {} + @masters_down.clear disable_readonly_mode end @@ -119,7 +125,6 @@ module ActiveRecord verify_replica(connection) else begin - now = Time.zone.now connection = postgresql_connection(config) fallback_handler.master_down = false rescue PG::ConnectionBad => e diff --git a/spec/components/active_record/connection_adapters/postgresql_fallback_adapter_spec.rb b/spec/components/active_record/connection_adapters/postgresql_fallback_adapter_spec.rb index 54a51d40f2a..efec0c5d54e 100644 --- a/spec/components/active_record/connection_adapters/postgresql_fallback_adapter_spec.rb +++ b/spec/components/active_record/connection_adapters/postgresql_fallback_adapter_spec.rb @@ -61,8 +61,6 @@ describe ActiveRecord::ConnectionHandling do end it 'should failover to a replica server' do - current_threads = Thread.list - RailsMultisite::ConnectionManagement.stubs(:all_dbs).returns(['default', multisite_db]) [config, multisite_config].each do |configuration| @@ -104,11 +102,10 @@ describe ActiveRecord::ConnectionHandling do postgresql_fallback_handler.initiate_fallback_to_master expect(Discourse.readonly_mode?).to eq(false) - expect(postgresql_fallback_handler.master_down?).to eq(nil) + expect(postgresql_fallback_handler.masters_down.hash).to eq({}) expect(Sidekiq.paused?).to eq(false) expect(ActiveRecord::Base.connection_pool.connections.count).to eq(0) - skip("Need to figure out why we keep running out of connections") expect(ActiveRecord::Base.connection) .to be_an_instance_of(ActiveRecord::ConnectionAdapters::PostgreSQLAdapter) end