Examples of org.apache.lucene.analysis.Token.term()

org.apache.lucene.analysis.Token.term()
Returns the Token's term text. This method has a performance penalty because the text is stored internally in a char[]. If possible, use {@link #termBuffer()} and {@link #termLength()} directly instead. If you really need aString, use this method, which is nothing more than a convenience call to new String(token.termBuffer(), 0, token.termLength())

        Reader r = new StringReader(text);
        TokenStream ts = index.getTextAnalyzer().tokenStream("", r);
        Token t = new Token();
        try {
            while ((t = ts.next(t)) != null) {
                String termText = t.term();
                TermVectorOffsetInfo[] info = termMap.get(termText);
                if (info == null) {
                    info = new TermVectorOffsetInfo[1];
                } else {
                    TermVectorOffsetInfo[] tmp = info;

View Full Code Here

      Token t = new Token();
      try
      {
         while ((t = ts.next(t)) != null)
         {
            String termText = t.term();
            TermVectorOffsetInfo[] info = (TermVectorOffsetInfo[])termMap.get(termText);
            if (info == null)
            {
               info = new TermVectorOffsetInfo[1];
            }

View Full Code Here

        token.setStartOffset(savedStart);
        token.setEndOffset(savedEnd);
        return reusableToken;
      } else
        while ((token = this.input.next(reusableToken)) != null) {
          if (token.term().equals("phrase")) {
            inPhrase = true;
            savedStart = token.startOffset();
            savedEnd = token.endOffset();
            token.setTermBuffer("phrase1");
            token.setStartOffset(savedStart);

View Full Code Here

            savedEnd = token.endOffset();
            token.setTermBuffer("phrase1");
            token.setStartOffset(savedStart);
            token.setEndOffset(savedEnd);
            return token;
          } else if (!token.term().equals("stop"))
            return token;
        }


      return null;

View Full Code Here

      @Override
      public boolean incrementToken() throws IOException {
        if(iter.hasNext()) {
          Token token = (Token) iter.next();
          clearAttributes();
          termAtt.setTermBuffer(token.term());
          posIncrAtt.setPositionIncrement(token.getPositionIncrement());
          offsetAtt.setOffset(token.startOffset(), token.endOffset());
          return true;
        }
        return false;

View Full Code Here

      @Override
      public boolean incrementToken() throws IOException {
        if(iter.hasNext()) {
          Token token = (Token) iter.next();
          clearAttributes();
          termAtt.setTermBuffer(token.term());
          posIncrAtt.setPositionIncrement(token.getPositionIncrement());
          offsetAtt.setOffset(token.startOffset(), token.endOffset());
          return true;
        }
        return false;

View Full Code Here

                        Token token = new Token();


                while (((token = tokenStream.next()) != null)
                                && (returnSet.size() < maximumNumberOfTokensToReturn)) {
                        
                                        returnSet.add(token.term());
                        
                        }
                        
                } catch (IOException e) {
                        log.severe(e.getMessage());

View Full Code Here

  @Override
  public Token next(Token token) throws IOException {
    if(tokenQueue.size() == 0) return null;
    if(token == null) token = new Token();
    Token fromQueue = tokenQueue.poll();
    token.reinit(fromQueue.term(), fromQueue.startOffset(), fromQueue.endOffset());
    return token;
  }
  
  @Override
  public Token next() throws IOException {

View Full Code Here

    );
    
    Token t = wdf.next();
    
    assertNotNull(t);
    assertEquals("übelkeit", t.term());
    assertEquals(7, t.startOffset());
    assertEquals(15, t.endOffset());
  }
  
  public void testOffsetChange2() throws Exception

View Full Code Here

    );
    
    Token t = wdf.next();
    
    assertNotNull(t);
    assertEquals("übelkeit", t.term());
    assertEquals(8, t.startOffset());
    assertEquals(17, t.endOffset());
  }
  
  public void testOffsetChange3() throws Exception

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.