/*
* Cloud9: A Hadoop toolkit for working with big data
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You may
* obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package edu.umd.cloud9.collection.trec;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
import com.google.common.base.Preconditions;
import edu.umd.cloud9.collection.DocumentForwardIndex;
/**
* A document forward index for TREC collections.
*
* @author Jimmy Lin
*/
public class TrecForwardIndex implements DocumentForwardIndex<TrecDocument> {
private static final Logger LOG = Logger.getLogger(TrecForwardIndex.class);
private long[] offsets;
private int[] lengths;
private FSDataInputStream input;
private TrecDocnoMapping docnoMapping = new TrecDocnoMapping();
private String path;
@Override
public int getDocno(String docid) {
Preconditions.checkNotNull(docid);
return docnoMapping.getDocno(docid);
}
@Override
public String getDocid(int docno) {
return docnoMapping.getDocid(docno);
}
@Override
public int getLastDocno() {
return offsets.length - 1;
}
@Override
public int getFirstDocno() {
return 1;
}
@Override
public String getCollectionPath() {
return path;
}
@Override
public TrecDocument getDocument(String docid) {
return getDocument(docnoMapping.getDocno(docid));
}
@Override
public TrecDocument getDocument(int docno) {
TrecDocument doc = new TrecDocument();
try {
LOG.info(String.format("docno %d: byte offset %d, length %d",
docno, offsets[docno], lengths[docno]));
input.seek(offsets[docno]);
byte[] arr = new byte[lengths[docno]];
input.read(arr);
TrecDocument.readDocument(doc, new String(arr));
} catch (IOException e) {
e.printStackTrace();
}
return doc;
}
@Override
public void loadIndex(Path index, Path mapping, FileSystem fs) throws IOException {
FSDataInputStream in = fs.open(index);
// Read and throw away.
in.readUTF();
path = in.readUTF();
// Docnos start at one, so we need an array that's one larger than number of docs.
int sz = in.readInt() + 1;
offsets = new long[sz];
lengths = new int[sz];
for (int i = 1; i < sz; i++) {
offsets[i] = in.readLong();
lengths[i] = in.readInt();
}
in.close();
input = fs.open(new Path(path));
docnoMapping.loadMapping(mapping, fs);
}
/**
* Simple program the provides access to the document contents.
*/
public static void main(String[] args) throws IOException {
if (args.length < 4) {
System.out.println("usage: [findex] [mapping-file] [getDocno|getDocid] [docid/docno]");
System.exit(-1);
}
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
System.out.println("forward index: " + args[0]);
System.out.println("mapping file: " + args[1]);
TrecForwardIndex findex = new TrecForwardIndex();
findex.loadIndex(new Path(args[0]), new Path(args[1]), fs);
if (args[2].equals("getDocno")) {
System.out.println("looking up docno " + args[3]);
System.out.println(findex.getDocument(Integer.parseInt(args[3])).getDisplayContent());
} else if (args[2].equals("getDocid")) {
System.out.println("looking up docid " + args[3]);
System.out.println(findex.getDocument(args[3]).getDisplayContent());
} else {
System.out.println("Invalid command!");
System.out.println("usage: [findex] [mapping-file] [getDocno|getDocid] [docid/docno]");
}
}
}