1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
|
// Copyright 2016 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.runtime;
import com.google.common.base.Preconditions;
import com.google.devtools.common.options.OptionsParsingException;
import com.sun.management.GarbageCollectionNotificationInfo;
import java.lang.management.GarbageCollectorMXBean;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryUsage;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Logger;
import javax.management.Notification;
import javax.management.NotificationEmitter;
import javax.management.NotificationListener;
import javax.management.openmbean.CompositeData;
/**
* Monitor the size of the retained heap and exit promptly if it grows too large. Specifically,
* check the size of the tenured space after each major GC; if it exceeds 90%, call
* {@code System.gc()} to trigger a stop-the-world collection; if it's still more than 90% full,
* exit with an {@link OutOfMemoryError}.
*/
class RetainedHeapLimiter implements NotificationListener {
private static final Logger logger = Logger.getLogger(RetainedHeapLimiter.class.getName());
private static final long MIN_TIME_BETWEEN_TRIGGERED_GC_MILLISECONDS = 60000;
private static int registeredOccupiedHeapPercentageThreshold = -1;
static void maybeInstallRetainedHeapLimiter(int occupiedHeapPercentageThreshold)
throws OptionsParsingException {
if (registeredOccupiedHeapPercentageThreshold == -1) {
registeredOccupiedHeapPercentageThreshold = occupiedHeapPercentageThreshold;
new RetainedHeapLimiter(occupiedHeapPercentageThreshold).install();
}
if (registeredOccupiedHeapPercentageThreshold != occupiedHeapPercentageThreshold) {
throw new OptionsParsingException(
"Old threshold of "
+ registeredOccupiedHeapPercentageThreshold
+ " not equal to new threshold of "
+ occupiedHeapPercentageThreshold
+ ". To change the threshold, shut down the server and restart it with the desired "
+ "value");
}
}
private boolean installed = false;
private final AtomicBoolean throwingOom = new AtomicBoolean(false);
private long lastTriggeredGcInMilliseconds = 0;
private final int occupiedHeapPercentageThreshold;
RetainedHeapLimiter(int occupiedHeapPercentageThreshold) {
this.occupiedHeapPercentageThreshold = occupiedHeapPercentageThreshold;
}
void install() {
Preconditions.checkState(!installed, "RetainedHeapLimiter installed twice");
installed = true;
List<GarbageCollectorMXBean> gcbeans = ManagementFactory.getGarbageCollectorMXBeans();
boolean foundTenured = false;
// Examine all collectors and register for notifications from those which collect the tenured
// space. Normally there is one such collector.
for (GarbageCollectorMXBean gcbean : gcbeans) {
boolean collectsTenured = false;
for (String name : gcbean.getMemoryPoolNames()) {
collectsTenured |= isTenuredSpace(name);
}
if (collectsTenured) {
foundTenured = true;
NotificationEmitter emitter = (NotificationEmitter) gcbean;
emitter.addNotificationListener(this, null, null);
}
}
if (!foundTenured) {
throw new IllegalStateException(
"Can't find tenured space; update this class for a new collector");
}
}
@Override
public void handleNotification(Notification notification, Object handback) {
if (!notification
.getType()
.equals(GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION)) {
return;
}
GarbageCollectionNotificationInfo info =
GarbageCollectionNotificationInfo.from((CompositeData) notification.getUserData());
Map<String, MemoryUsage> spaces = info.getGcInfo().getMemoryUsageAfterGc();
for (Map.Entry<String, MemoryUsage> entry : spaces.entrySet()) {
if (isTenuredSpace(entry.getKey())) {
MemoryUsage space = entry.getValue();
if (space.getMax() == 0) {
// The CMS collector sometimes passes us nonsense stats.
continue;
}
long percentUsed = 100 * space.getUsed() / space.getMax();
if (percentUsed > occupiedHeapPercentageThreshold) {
if (info.getGcCause().equals("System.gc()") && !throwingOom.getAndSet(true)) {
// Assume we got here from a GC initiated by the other branch.
String exitMsg =
String.format(
"RetainedHeapLimiter forcing exit due to GC thrashing: tenured space "
+ "%s out of %s (>%s%%) occupied after back-to-back full GCs",
space.getUsed(),
space.getMax(),
occupiedHeapPercentageThreshold);
System.err.println(exitMsg);
logger.info(exitMsg);
// Exits the runtime.
BugReport.handleCrash(new OutOfMemoryError(exitMsg));
} else if (System.currentTimeMillis() - lastTriggeredGcInMilliseconds
> MIN_TIME_BETWEEN_TRIGGERED_GC_MILLISECONDS) {
logger.info(
"Triggering a full GC with "
+ space.getUsed()
+ " out of "
+ space.getMax()
+ " used");
// Force a full stop-the-world GC and see if it can get us below the threshold.
System.gc();
lastTriggeredGcInMilliseconds = System.currentTimeMillis();
}
}
}
}
}
private static boolean isTenuredSpace(String name) {
return "CMS Old Gen".equals(name)
|| "G1 Old Gen".equals(name)
|| "PS Old Gen".equals(name)
|| "Tenured Gen".equals(name);
}
}
|
