Examples of org.apache.lucene.analysis.Token.startOffset()

org.apache.lucene.analysis.Token.startOffset()
Returns this Token's starting offset, the position of the first character corresponding to this token in the source text. Note that the difference between endOffset() and startOffset() may not be equal to {@link #termLength}, as the term text may have been altered by a stemmer or some other filter.

      StringBuilder collation = new StringBuilder(origQuery);
      int offset = 0;
      for (Iterator<Map.Entry<Token, String>> bestIter = best.entrySet().iterator(); bestIter.hasNext();) {
        Map.Entry<Token, String> entry = bestIter.next();
        Token tok = entry.getKey();
        collation.replace(tok.startOffset() + offset, 
          tok.endOffset() + offset, entry.getValue());
        offset += entry.getValue().length() - (tok.endOffset() - tok.startOffset());
      }
      String collVal = collation.toString();
      if (collVal.equals(origQuery) == false) {

View Full Code Here

      for (Iterator<Map.Entry<Token, String>> bestIter = best.entrySet().iterator(); bestIter.hasNext();) {
        Map.Entry<Token, String> entry = bestIter.next();
        Token tok = entry.getKey();
        collation.replace(tok.startOffset() + offset, 
          tok.endOffset() + offset, entry.getValue());
        offset += entry.getValue().length() - (tok.endOffset() - tok.startOffset());
      }
      String collVal = collation.toString();
      if (collVal.equals(origQuery) == false) {
        LOG.fine("Collation:" + collation);
        result.add("collation", collVal);

View Full Code Here

    Token t = null;
    while (((t = tstream.next()) != null)) {
      NamedList<Object> token = new SimpleOrderedMap<Object>();
      tokens.add("token", token);
      token.add("value", new String(t.termBuffer(), 0, t.termLength()));
      token.add("start", t.startOffset());
      token.add("end", t.endOffset());
      token.add("posInc", t.getPositionIncrement());
      token.add("type", t.type());
      //TODO: handle payloads
    }

View Full Code Here

          // Now grab the hit-element, if present
          //
          Token t = tokens[j];
          if (highlight.contains(t.termText())) {
            excerpt.addToken(t.termText());
            excerpt.add(new Fragment(text.substring(offset, t.startOffset())));
            excerpt.add(new Highlight(text.substring(t.startOffset(),t.endOffset())));
            offset = t.endOffset();
            endToken = Math.min(j + sumContext, tokens.length);
          }

View Full Code Here

          //
          Token t = tokens[j];
          if (highlight.contains(t.termText())) {
            excerpt.addToken(t.termText());
            excerpt.add(new Fragment(text.substring(offset, t.startOffset())));
            excerpt.add(new Highlight(text.substring(t.startOffset(),t.endOffset())));
            offset = t.endOffset();
            endToken = Math.min(j + sumContext, tokens.length);
          }
          
          j++;

View Full Code Here

                StringBuffer sb = new StringBuffer(statement);
                for (int i = suggestions.length - 1; i >= 0; i--) {
                    Token t = (Token) tokens.get(i);
                    // only replace if word actually changed
                    if (!t.termText().equalsIgnoreCase(suggestions[i])) {
                        sb.replace(t.startOffset(), t.endOffset(), suggestions[i]);
                    }
                }
                return sb.toString();
            } else {
                return null;

View Full Code Here

        private void tokenize(String statement, List<String> words, List<Token> tokens) throws IOException {
            TokenStream ts = handler.getTextAnalyzer().tokenStream(FieldNames.FULLTEXT, new StringReader(statement));
            try {
                Token t;
                while ((t = ts.next()) != null) {
                    String origWord = statement.substring(t.startOffset(), t.endOffset());
                    if (t.getPositionIncrement() > 0) {
                        words.add(t.termText());
                        tokens.add(t);
                    } else {
                        // very simple implementation: use termText with length

View Full Code Here

          // Now grab the hit-element, if present
          //
          Token t = tokens[j];
          if (highlight.contains(t.termText())) {
            excerpt.addToken(t.termText());
            excerpt.add(new Fragment(text.substring(offset, t.startOffset())));
            excerpt.add(new Highlight(text.substring(t.startOffset(),t.endOffset())));
            offset = t.endOffset();
            endToken = Math.min(j+SUM_CONTEXT, tokens.length);
          }

View Full Code Here

          //
          Token t = tokens[j];
          if (highlight.contains(t.termText())) {
            excerpt.addToken(t.termText());
            excerpt.add(new Fragment(text.substring(offset, t.startOffset())));
            excerpt.add(new Highlight(text.substring(t.startOffset(),t.endOffset())));
            offset = t.endOffset();
            endToken = Math.min(j+SUM_CONTEXT, tokens.length);
          }


          j++;

View Full Code Here

          // Now grab the hit-element, if present
          //
          Token t = tokens[j];
          if (highlight.contains(t.termText())) {
            excerpt.addToken(t.termText());
            excerpt.add(new Fragment(text.substring(offset, t.startOffset())));
            excerpt.add(new Highlight(text.substring(t.startOffset(),t.endOffset())));
            offset = t.endOffset();
            endToken = Math.min(j+SUM_CONTEXT, tokens.length);
          }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.