this.token = token;
this.index = index;
this.inChunk = chunk;
this.hasAlphaNumeric = Utils.hasAlphaNumericChar(token.getSpan());
this.hasSearchableLength = token.getSpan().length() >= tpc.getMinSearchTokenLength();
PosTag selectedPosTag = null;
boolean matchedPosTag = false; //matched any of the POS annotations
//(1) check if this Token should be linked against the Vocabulary (isProcessable)
upperCase = token.getEnd() > token.getStart() && //not an empty token
Character.isUpperCase(token.getSpan().codePointAt(0)); //and upper case
boolean isLinkablePos = false;
boolean isMatchablePos = false;
boolean isSubSentenceStart = false;
List<Value<PosTag>> posAnnotations = token.getAnnotations(POS_ANNOTATION);
for(Value<PosTag> posAnnotation : posAnnotations){
// check three possible match
// 1. the LexicalCategory matches
// 2. the Pos matches
// 3. the String tag matches
PosTag posTag = posAnnotation.value();
if((!disjoint(tpc.getLinkedLexicalCategories(), posTag.getCategories())) ||
(!disjoint(tpc.getLinkedPos(), posTag.getPosHierarchy())) ||
tpc.getLinkedPosTags().contains(posTag.getTag())){
if(posAnnotation.probability() == Value.UNKNOWN_PROBABILITY ||
posAnnotation.probability() >= tpc.getMinPosAnnotationProbability()){
selectedPosTag = posTag;
isLinkablePos = true;
isMatchablePos = true;
matchedPosTag = true;
break;
} // else probability to low for inclusion
} else if(posAnnotation.probability() == Value.UNKNOWN_PROBABILITY ||
posAnnotation.probability() >= tpc.getMinExcludePosAnnotationProbability()){
selectedPosTag = posTag; //also rejected PosTags are selected
matchedPosTag = true;
isLinkablePos = false;
break;
} // else probability to low for exclusion
}
if(!matchedPosTag) { //not matched against a POS Tag ...
this.isLinkablePos = null;
} else {
this.isLinkablePos = isLinkablePos;
}
//(2) check if this token should be considered to match labels of suggestions
if(this.isLinkablePos != null && this.isLinkablePos){ //processable tokens are also matchable
this.isMatchablePos = true;
} else { //check POS and length to see if token is matchable
matchedPosTag = false; //reset to false!
for(Value<PosTag> posAnnotation : posAnnotations){
PosTag posTag = posAnnotation.value();
if(posTag.isMapped()){
if(!Collections.disjoint(tpc.getMatchedLexicalCategories(),
posTag.getCategories())){
if(posAnnotation.probability() == Value.UNKNOWN_PROBABILITY ||
posAnnotation.probability() >= tpc.getMinPosAnnotationProbability()){
//override selectedPosTag if present
selectedPosTag = posTag; //mark the matchable as selected PosTag
isMatchablePos = true;
matchedPosTag = true;
break;
} // else probability to low for inclusion
} else if(posAnnotation.probability() == Value.UNKNOWN_PROBABILITY ||
posAnnotation.probability() >= tpc.getMinExcludePosAnnotationProbability()){
if(selectedPosTag == null){ //do not override existing values
selectedPosTag = posTag; //also rejected PosTags are selected
}
isMatchablePos = false;
matchedPosTag = true;
break;
} // else probability to low for exclusion
} //else not matched ... search next one
}
if(!matchedPosTag){ //not matched against POS tag ...
//fall back to the token length
this.isMatchablePos = null;
//this.isMatchablePos = token.getSpan().length() >= tpc.getMinSearchTokenLength();
} else {
this.isMatchablePos = isMatchablePos;
}
}
//(3) check if the POS tag indicates the start/end of an sub-sentence
for(Value<PosTag> posAnnotation : posAnnotations){
PosTag posTag = posAnnotation.value();
if((!disjoint(ProcessingState.SUB_SENTENCE_START_POS,posTag.getPosHierarchy()))){
if(posAnnotation.probability() == Value.UNKNOWN_PROBABILITY ||
posAnnotation.probability() >= tpc.getMinPosAnnotationProbability()){
isSubSentenceStart = true;
} // else probability to low for inclusion
} else if(posAnnotation.probability() == Value.UNKNOWN_PROBABILITY ||
posAnnotation.probability() >= tpc.getMinExcludePosAnnotationProbability()){
isSubSentenceStart = false;
}
}
this.isSubSentenceStart = isSubSentenceStart;
//(4) check for morpho analyses
if(selectedPosTag == null){ //token is not processable or matchable
//we need to set the selectedPoas tag to the first POS annotation
Value<PosTag> posAnnotation = token.getAnnotation(POS_ANNOTATION);
if(posAnnotation != null) {
selectedPosTag = posAnnotation.value();
}
}
List<Value<MorphoFeatures>> morphoAnnotations = token.getAnnotations(NlpAnnotations.MORPHO_ANNOTATION);
if(selectedPosTag == null){ //no POS information ... use the first morpho annotation
morpho = morphoAnnotations.isEmpty() ? null : morphoAnnotations.get(0).value();
} else { //select the correct morpho annotation based on the POS tag
MorphoFeatures mf = null;
selectMorphoFeature :
for(Value<MorphoFeatures> morphoAnnotation : morphoAnnotations){
for(PosTag posTag : morphoAnnotation.value().getPosList()){
if(!disjoint(selectedPosTag.getCategories(),posTag.getCategories())){
mf = morphoAnnotation.value();
break selectMorphoFeature; //stop after finding the first one
}
}
}