String input = cmdline.getOptionValue(INPUT);
String output = cmdline.getOptionValue(OUTPUT);
List<String> stringList = Lists.newArrayList();
IntArrayList intList = new IntArrayList();
// First read lines into sorted map to sort input.
Object2IntAVLTreeMap<String> tree = new Object2IntAVLTreeMap<String>();
BufferedReader br = new BufferedReader(new FileReader(input));
String line;
while ((line = br.readLine()) != null) {
String[] arr = line.split("\\t");
if ( arr[0] == null || arr[0].length() == 0) {
LOG.info("Skipping invalid line: " + line);
}
tree.put(arr[0], Integer.parseInt(arr[1]));
}
br.close();
// Extracted sorted strings and ints.
for (Object2IntMap.Entry<String> map : tree.object2IntEntrySet()) {
stringList.add(map.getKey());
intList.add(map.getIntValue());
}
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
FSDataOutputStream os = fs.create(new Path(output), true);
ByteArrayOutputStream bytesOut;
ObjectOutputStream objOut;
byte[] bytes;
// Serialize the front-coded dictionary
FrontCodedStringList frontcodedList = new FrontCodedStringList(stringList, 8, true);
bytesOut = new ByteArrayOutputStream();
objOut = new ObjectOutputStream(bytesOut);
objOut.writeObject(frontcodedList);
objOut.close();
bytes = bytesOut.toByteArray();
os.writeInt(bytes.length);
os.write(bytes);
// Serialize the hash function
ShiftAddXorSignedStringMap dict = new ShiftAddXorSignedStringMap(stringList.iterator(),
new TwoStepsLcpMonotoneMinimalPerfectHashFunction<CharSequence>(stringList,
TransformationStrategies.prefixFreeUtf16()));
bytesOut = new ByteArrayOutputStream();
objOut = new ObjectOutputStream(bytesOut);
objOut.writeObject(dict);
objOut.close();
bytes = bytesOut.toByteArray();
os.writeInt(bytes.length);
os.write(bytes);
// Serialize the ints.
os.writeInt(intList.size());
for (int i = 0; i < intList.size(); i++) {
os.writeInt(intList.getInt(i));
}
os.close();
}