String controllerName = CONTROLLER_PREFIX + "_0";
StartCMResult startResult =
_startCMResultMap.get(controllerName);
HelixDataAccessor accessor = startResult._manager.getHelixDataAccessor();
IdealState idealState = _setupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, TEST_DB);
Map<String, ZNRecord> scnTableMap = new HashMap<String, ZNRecord>();
for (int i = 0; i < NODE_NR; i++)
{
String instance = PARTICIPANT_PREFIX + "_" + (START_PORT + i);
ZNRecord scnRecord = new ZNRecord("scnTable");
scnRecord.setSimpleField("instance", instance);
scnTableMap.put(instance, scnRecord);
}
String instanceDead = PARTICIPANT_PREFIX + "_" + (START_PORT + 0);
for(int j = 0; j < _PARTITIONS; j++)
{
int seq = 50;
String partition = TEST_DB + "_" + j;
List<String> idealStatePrefList =
idealState.getPreferenceList(partition);
String idealStateMaster = idealStatePrefList.get(0);
// Switch the scn order of the partitions mastered on instanceDead
if(idealStateMaster.equals(instanceDead))
{
for(int x = 0; x < idealStatePrefList.size(); x++)
{
String instance = idealStatePrefList.get(x);
ZNRecord scnRecord = scnTableMap.get(instance);
if(!scnRecord.getMapFields().containsKey(partition))
{
scnRecord.setMapField(partition, new HashMap<String, String>());
}
Map<String, String> scnDetails = scnRecord.getMapField(partition);
scnDetails.put("gen", "4");
if(x > 0)
{
scnDetails.put("seq", "" + (seq - 22 + 11 *(x)));
}
else
{
scnDetails.put("seq", "100");
}
}
}
}
for(String instanceName : scnTableMap.keySet())
{
Builder kb = accessor.keyBuilder();
accessor.setProperty(kb.healthReport(instanceName, "scnTable"), new HealthStat(scnTableMap.get(instanceName)));
}
// kill a node, after a while the master should be the last one in the ideal state pref list
_startCMResultMap.get(instanceDead)._manager.disconnect();
_startCMResultMap.get(instanceDead)._thread.interrupt();
Thread.sleep(1000);
boolean verifyResult =
ClusterStateVerifier.verifyByZkCallback(new MasterNbInExtViewVerifier(ZK_ADDR,
CLUSTER_NAME));
Assert.assertTrue(verifyResult);
Builder kb = accessor.keyBuilder();
ExternalView ev = accessor.getProperty(kb.externalView(TEST_DB));
for(String partitionName : idealState.getPartitionSet())
{
List<String> prefList = idealState.getPreferenceList(partitionName);
if(prefList.get(0).equals(instanceDead))
{
String last = prefList.get(prefList.size() - 1);
Assert.assertTrue(ev.getStateMap(partitionName).get(last).equals("MASTER"));
}
}
// Bring up the previous dead node, but as the SCN is the last for all the
// master partitions on it, the master partitions should be still on the last if the prefList
StartCMResult result =
TestHelper.startDummyProcess(ZK_ADDR, CLUSTER_NAME, instanceDead);
_startCMResultMap.put(instanceDead, result);
Thread.sleep(1000);
verifyResult =
ClusterStateVerifier.verifyByZkCallback(new MasterNbInExtViewVerifier(ZK_ADDR,
CLUSTER_NAME));
Assert.assertTrue(verifyResult);
for(String partitionName : idealState.getPartitionSet())
{
List<String> prefList = idealState.getPreferenceList(partitionName);
if(prefList.get(0).equals(instanceDead))
{
String last = prefList.get(prefList.size() - 1);
Assert.assertTrue(ev.getStateMap(partitionName).get(last).equals("MASTER"));
}
}
// Reset the scn of the partitions
for(int j = 0; j < _PARTITIONS; j++)
{
String partition = TEST_DB + "_" + j;
List<String> idealStatePrefList =
idealState.getPreferenceList(partition);
String idealStateMaster = idealStatePrefList.get(0);
// Switch back the scn to the same
if(idealStateMaster.equals(instanceDead))
{
for(int x = 0; x < idealStatePrefList.size(); x++)
{
String instance = idealStatePrefList.get(x);
ZNRecord scnRecord = scnTableMap.get(instance);
if(!scnRecord.getMapFields().containsKey(partition))
{
scnRecord.setMapField(partition, new HashMap<String, String>());
}
Map<String, String> scnDetails = scnRecord.getMapField(partition);
scnDetails.put("gen", "4");
scnDetails.put("seq", "100");
}
}
}
// set the scn to normal -- same order as the priority list
for(String instanceName : scnTableMap.keySet())
{
kb = accessor.keyBuilder();
accessor.setProperty(kb.healthReport(instanceName, "scnTable"), new HealthStat(scnTableMap.get(instanceName)));
}
Thread.sleep(1000);
verifyResult =
ClusterStateVerifier.verifyByZkCallback(new MasterNbInExtViewVerifier(ZK_ADDR,
CLUSTER_NAME));
Assert.assertTrue(verifyResult);
// should be reverted to normal
ev = accessor.getProperty(kb.externalView(TEST_DB));
for(String partitionName : idealState.getPartitionSet())
{
List<String> prefList = idealState.getPreferenceList(partitionName);
if(prefList.get(0).equals(instanceDead))
{
String last = prefList.get(prefList.size() - 1);
Assert.assertTrue(ev.getStateMap(partitionName).get(last).equals("SLAVE"));
Assert.assertTrue(ev.getStateMap(partitionName).get(prefList.get(1)).equals("SLAVE"));