if (token == null
|| token.length() == 0
|| queryTokenizer.inTagToSkip())
continue;
if (queryTokenizer.inDocnoTag()) {
//The tokenizer is constructed from the trimmed version of the contents
//of the query number tag, so that the last token extracted from it, is
//always the query number, and not an empty string
StringTokenizer docnoTokens =
new StringTokenizer(token.trim(), " ");