Examples of org.apache.lucene.analysis.Token.termBuffer()

org.apache.lucene.analysis.Token.termBuffer()
Returns the internal termBuffer character array which you can then directly alter. If the array is too small for your token, use {@link #resizeTermBuffer(int)} to increase it. Afteraltering the buffer be sure to call {@link #setTermLength} to record the number of validcharacters that were placed into the termBuffer.

  }


  @Override
  public final Token next(Token in) throws IOException {
    Token t = input.next(in);
    if (null == t || null == t.termBuffer() || t.termLength() == 0){
      return t;
    }
    char[] termBuffer = t.termBuffer();
    int len = t.termLength();
    int start = 0;

View Full Code Here

  public final Token next(Token in) throws IOException {
    Token t = input.next(in);
    if (null == t || null == t.termBuffer() || t.termLength() == 0){
      return t;
    }
    char[] termBuffer = t.termBuffer();
    int len = t.termLength();
    int start = 0;
    int end = 0;
    int endOff = 0;

View Full Code Here

    for (end = len; end >= start && termBuffer[end - 1] <= ' '; end--) {
      endOff++;
    }
    if (start > 0 || end < len) {
      if (start < end) {
        t.setTermBuffer(t.termBuffer(), start, (end - start));
      } else {
        t.setTermLength(0);
      }
      if (updateOffsets) {
        t.setStartOffset(t.startOffset() + start);

View Full Code Here


  @Override
  public Token next(Token token) throws IOException {
    Token result = input.next(token);
    if (result != null) {
      char[] termBuffer = result.termBuffer();
      int len = result.termLength();
      // if protected, don't stem.  use this to avoid stemming collisions.
      if (protWords != null && protWords.contains(termBuffer, 0, len)) {
        return result;
      }

View Full Code Here

          best.put(inputToken, theSuggestions.keySet().iterator().next());
        }
        if (hasFreqInfo) {
          isCorrectlySpelled = isCorrectlySpelled && spellingResult.getTokenFrequency(inputToken) > 0;
        }
        result.add(new String(inputToken.termBuffer(), 0, inputToken.termLength()), suggestionList);
      }
    }
    if (hasFreqInfo) {
      result.add("correctlySpelled", isCorrectlySpelled);
    }

View Full Code Here

  
  public final Token next(Token in) throws IOException {
    Token t = input.next(in);
    if (t == null)
      return null;
    CharSequence text = CharBuffer.wrap(t.termBuffer(), 0, t.termLength());
    Matcher m = p.matcher(text);
    if (all) {
      t.setTermText(m.replaceAll(replacement));
    } else {
      t.setTermText(m.replaceFirst(replacement));

View Full Code Here

        t = null;
      }
      boolean dup=false;
      for (Token outTok : output()) {
        int tokLen = tok.termLength();
        if (outTok.termLength() == tokLen && ArraysUtils.equals(outTok.termBuffer(), 0, tok.termBuffer(), 0, tokLen)) {
          dup=true;
          //continue;;
        }
      }
      if (!dup){

View Full Code Here

      return temp;
    }
    
    Token t = input.next(in);
    if( t != null ) {
      String value = new String(t.termBuffer(), 0, t.termLength());
      try {
        value = encoder.encode(value).toString();
      } 
      catch (Exception ignored) {} // just use the direct text
      //Token m = new Token(value, t.startOffset(), t.endOffset(), name );

View Full Code Here

    NamedList<NamedList<Object>> tokens = new NamedList<NamedList<Object>>();
    Token t = null;
    while (((t = tstream.next()) != null)) {
      NamedList<Object> token = new SimpleOrderedMap<Object>();
      tokens.add("token", token);
      token.add("value", new String(t.termBuffer(), 0, t.termLength()));
      token.add("start", t.startOffset());
      token.add("end", t.endOffset());
      token.add("posInc", t.getPositionIncrement());
      token.add("type", t.type());
      //TODO: handle payloads

View Full Code Here

      // t is either returned, or a new token is made from it, so it should
      // be safe to use the next(Token) method.
      t = input.next(in);
      if (t == null) return null;


      char [] termBuffer = t.termBuffer();
      int len = t.termLength();
      int start=0;
      if (len ==0) continue;


      origPosIncrement += t.getPositionIncrement();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.