Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.startOffset()


  @Override
  public Token next(Token token) throws IOException {
    if(tokenQueue.size() == 0) return null;
    if(token == null) token = new Token();
    Token fromQueue = tokenQueue.poll();
    token.reinit(fromQueue.term(), fromQueue.startOffset(), fromQueue.endOffset());
    return token;
  }
 
  @Override
  public Token next() throws IOException {
View Full Code Here


    int pos=0;
    for (Iterator iter = a.iterator(); iter.hasNext();) {
      Token tok = (Token)iter.next();
      pos += tok.getPositionIncrement();
      if (!tokAt(b, new String(tok.termBuffer(), 0, tok.termLength()), pos
              , checkOff ? tok.startOffset() : -1
              , checkOff ? tok.endOffset() : -1
              ))
      {
        fail(a + "!=" + b);
      }
View Full Code Here

    int pos=0;
    for (Iterator iter = lst.iterator(); iter.hasNext();) {
      Token tok = (Token)iter.next();
      pos += tok.getPositionIncrement();
      if (pos==tokPos && new String(tok.termBuffer(), 0, tok.termLength()).equals(val)
          && (startOff==-1 || tok.startOffset()==startOff)
          && (endOff  ==-1 || tok.endOffset()  ==endOff  )
           )
      {
        return true;
      }
View Full Code Here

   
    Token t = wdf.next();
   
    assertNotNull(t);
    assertEquals("übelkeit", t.term());
    assertEquals(7, t.startOffset());
    assertEquals(15, t.endOffset());
  }
 
  public void testOffsetChange2() throws Exception
  {
View Full Code Here

   
    Token t = wdf.next();
   
    assertNotNull(t);
    assertEquals("übelkeit", t.term());
    assertEquals(8, t.startOffset());
    assertEquals(17, t.endOffset());
  }
 
  public void testOffsetChange3() throws Exception
  {
View Full Code Here

   
    Token t = wdf.next();
   
    assertNotNull(t);
    assertEquals("übelkeit", t.term());
    assertEquals(8, t.startOffset());
    assertEquals(16, t.endOffset());
  }
 
  public void testOffsetChange4() throws Exception
  {
View Full Code Here

   
    Token t = wdf.next();
   
    assertNotNull(t);
    assertEquals("foo", t.term());
    assertEquals(8, t.startOffset());
    assertEquals(11, t.endOffset());
   
    t = wdf.next();
   
    assertNotNull(t);
View Full Code Here

   
    t = wdf.next();
   
    assertNotNull(t);
    assertEquals("bar", t.term());
    assertEquals(12, t.startOffset());
    assertEquals(15, t.endOffset());
  }

  public void testAlphaNumericWords(){
     assertU(adoc("id""68","numericsubword","Java/J2SE"));
View Full Code Here

        t.setTermBuffer(t.termBuffer(), start, (end - start));
      } else {
        t.setTermLength(0);
      }
      if (updateOffsets) {
        t.setStartOffset(t.startOffset() + start);
        if (start < end) {
          t.setEndOffset(t.endOffset() - endOff);
        } //else if end is less than, start, then the term length is 0, so, no need to bother w/ the end offset
      }
      /*t = new Token( t.termText().substring( start, end ),
View Full Code Here

      Token inputToken = entry.getKey();
      Map<String, Integer> theSuggestions = entry.getValue();
      if (theSuggestions != null && theSuggestions.size() > 0) {
        SimpleOrderedMap suggestionList = new SimpleOrderedMap();
        suggestionList.add("numFound", theSuggestions.size());
        suggestionList.add("startOffset", inputToken.startOffset());
        suggestionList.add("endOffset", inputToken.endOffset());
        if (extendedResults && hasFreqInfo) {
          suggestionList.add("origFreq", spellingResult.getTokenFrequency(inputToken));
          for (Map.Entry<String, Integer> suggEntry : theSuggestions.entrySet()) {
            SimpleOrderedMap<Object> suggestionItem = new SimpleOrderedMap<Object>();
View Full Code Here

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.