synchronized (openingTablets) {
openingTablets.remove(extent);
openingTablets.notifyAll();
}
log.warn("Failed to verify tablet " + extent, e);
enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.LOAD_FAILURE, extent));
throw new RuntimeException(e);
}
if (locationToOpen == null) {
log.debug("Reporting tablet " + extent + " assignment failure: unable to verify Tablet Information");
synchronized (openingTablets) {
openingTablets.remove(extent);
openingTablets.notifyAll();
}
enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.LOAD_FAILURE, extent));
return;
}
Tablet tablet = null;
boolean successful = false;
try {
TabletResourceManager trm = resourceManager.createTabletResourceManager();
// this opens the tablet file and fills in the endKey in the
// extent
locationToOpen = VolumeUtil.switchRootTabletVolume(extent, locationToOpen);
tablet = new Tablet(TabletServer.this, locationToOpen, extent, trm, tabletsKeyValues);
/*
* If a minor compaction starts after a tablet opens, this indicates a log recovery occurred. This recovered data must be minor compacted.
*
* There are three reasons to wait for this minor compaction to finish before placing the tablet in online tablets.
*
* 1) The log recovery code does not handle data written to the tablet on multiple tablet servers. 2) The log recovery code does not block if memory is
* full. Therefore recovering lots of tablets that use a lot of memory could run out of memory. 3) The minor compaction finish event did not make it to
* the logs (the file will be in metadata, preventing replay of compacted data)... but do not want a majc to wipe the file out from metadata and then
* have another process failure... this could cause duplicate data to replay
*/
if (tablet.getNumEntriesInMemory() > 0 && !tablet.minorCompactNow(MinorCompactionReason.RECOVERY)) {
throw new RuntimeException("Minor compaction after recovery fails for " + extent);
}
Assignment assignment = new Assignment(extent, getTabletSession());
TabletStateStore.setLocation(assignment);
synchronized (openingTablets) {
synchronized (onlineTablets) {
openingTablets.remove(extent);
onlineTablets.put(extent, tablet);
openingTablets.notifyAll();
recentlyUnloadedCache.remove(tablet.getExtent());
}
}
tablet = null; // release this reference
successful = true;
} catch (Throwable e) {
log.warn("exception trying to assign tablet " + extent + " " + locationToOpen, e);
if (e.getMessage() != null)
log.warn(e.getMessage());
String table = extent.getTableId().toString();
ProblemReports.getInstance().report(new ProblemReport(table, TABLET_LOAD, extent.getUUID().toString(), getClientAddressString(), e));
}
if (!successful) {
synchronized (unopenedTablets) {
synchronized (openingTablets) {
openingTablets.remove(extent);
unopenedTablets.add(extent);
openingTablets.notifyAll();
}
}
log.warn("failed to open tablet " + extent + " reporting failure to master");
enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.LOAD_FAILURE, extent));
long reschedule = Math.min((1l << Math.min(32, retryAttempt)) * 1000, 10 * 60 * 1000l);
log.warn(String.format("rescheduling tablet load in %.2f seconds", reschedule / 1000.));
SimpleTimer.getInstance().schedule(new TimerTask() {
@Override
public void run() {
log.info("adding tablet " + extent + " back to the assignment pool (retry " + retryAttempt + ")");
AssignmentHandler handler = new AssignmentHandler(extent, retryAttempt + 1);
if (extent.isMeta()) {
if (extent.isRootTablet()) {
new Daemon(new LoggingRunnable(log, handler), "Root tablet assignment retry").start();
} else {
resourceManager.addMetaDataAssignment(handler);
}
} else {
resourceManager.addAssignment(handler);
}
}
}, reschedule);
} else {
enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.LOADED, extent));
}
}