1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
|
# frozen_string_literal: true
module ActionDispatch
# This middleware can be used to diagnose deadlocks in the autoload interlock.
#
# To use it, insert it near the top of the middleware stack, using
# <tt>config/application.rb</tt>:
#
# config.middleware.insert_before Rack::Sendfile, ActionDispatch::DebugLocks
#
# After restarting the application and re-triggering the deadlock condition,
# <tt>/rails/locks</tt> will show a summary of all threads currently known to
# the interlock, which lock level they are holding or awaiting, and their
# current backtrace.
#
# Generally a deadlock will be caused by the interlock conflicting with some
# other external lock or blocking I/O call. These cannot be automatically
# identified, but should be visible in the displayed backtraces.
#
# NOTE: The formatting and content of this middleware's output is intended for
# human consumption, and should be expected to change between releases.
#
# This middleware exposes operational details of the server, with no access
# control. It should only be enabled when in use, and removed thereafter.
class DebugLocks
def initialize(app, path = "/rails/locks")
@app = app
@path = path
end
def call(env)
req = ActionDispatch::Request.new env
if req.get?
path = req.path_info.chomp("/".freeze)
if path == @path
return render_details(req)
end
end
@app.call(env)
end
private
def render_details(req)
threads = ActiveSupport::Dependencies.interlock.raw_state do |threads|
# The Interlock itself comes to a complete halt as long as this block
# is executing. That gives us a more consistent picture of everything,
# but creates a pretty strong Observer Effect.
#
# Most directly, that means we need to do as little as possible in
# this block. More widely, it means this middleware should remain a
# strictly diagnostic tool (to be used when something has gone wrong),
# and not for any sort of general monitoring.
threads.each.with_index do |(thread, info), idx|
info[:index] = idx
info[:backtrace] = thread.backtrace
end
threads
end
str = threads.map do |thread, info|
if info[:exclusive]
lock_state = "Exclusive"
elsif info[:sharing] > 0
lock_state = "Sharing"
lock_state << " x#{info[:sharing]}" if info[:sharing] > 1
else
lock_state = "No lock"
end
if info[:waiting]
lock_state << " (yielded share)"
end
msg = "Thread #{info[:index]} [0x#{thread.__id__.to_s(16)} #{thread.status || 'dead'}] #{lock_state}\n"
if info[:sleeper]
msg << " Waiting in #{info[:sleeper]}"
msg << " to #{info[:purpose].to_s.inspect}" unless info[:purpose].nil?
msg << "\n"
if info[:compatible]
compat = info[:compatible].map { |c| c == false ? "share" : c.to_s.inspect }
msg << " may be pre-empted for: #{compat.join(', ')}\n"
end
blockers = threads.values.select { |binfo| blocked_by?(info, binfo, threads.values) }
msg << " blocked by: #{blockers.map { |i| i[:index] }.join(', ')}\n" if blockers.any?
end
blockees = threads.values.select { |binfo| blocked_by?(binfo, info, threads.values) }
msg << " blocking: #{blockees.map { |i| i[:index] }.join(', ')}\n" if blockees.any?
msg << "\n#{info[:backtrace].join("\n")}\n" if info[:backtrace]
end.join("\n\n---\n\n\n")
[200, { "Content-Type" => "text/plain", "Content-Length" => str.size }, [str]]
end
def blocked_by?(victim, blocker, all_threads)
return false if victim.equal?(blocker)
case victim[:sleeper]
when :start_sharing
blocker[:exclusive] ||
(!victim[:waiting] && blocker[:compatible] && !blocker[:compatible].include?(false))
when :start_exclusive
blocker[:sharing] > 0 ||
blocker[:exclusive] ||
(blocker[:compatible] && !blocker[:compatible].include?(victim[:purpose]))
when :yield_shares
blocker[:exclusive]
when :stop_exclusive
blocker[:exclusive] ||
victim[:compatible] &&
victim[:compatible].include?(blocker[:purpose]) &&
all_threads.all? { |other| !other[:compatible] || blocker.equal?(other) || other[:compatible].include?(blocker[:purpose]) }
end
end
end
end
|
