Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.type()

107
108
109
110
111
112
113
114
115
116
117
        Token token = tokens[index++];
        termAtt.setEmpty().append(token);
        offsetAtt.setOffset(token.startOffset(), token.endOffset());
        posIncAtt.setPositionIncrement(token.getPositionIncrement());
        flagsAtt.setFlags(token.getFlags());
        typeAtt.setType(token.type());
        payloadAtt.setPayload(token.getPayload());
        return true;
      }
    }
  }
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
      tokens.add("token", token);
      token.add("value", new String(t.termBuffer(), 0, t.termLength()));
      token.add("start", t.startOffset());
      token.add("end", t.endOffset());
      token.add("posInc", t.getPositionIncrement());
      token.add("type", t.type());
      //TODO: handle payloads
    }
    return tokens;
  }
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
     * "the_sound"|"sound", "sound_of" For case 1 we consume the gram from the
     * input stream and output it rather than the current token This means that
     * the call to super.next() which reads a token from input and passes it on
     * to this process method will always get a token of type word
     */
    if (next != null && next.type() == "gram") {
      // consume "next" token from list and output it
      token = read();
      // use this to clone the token because clone requires all these args but
      // won't take the token.type
      // see
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
      tokens.add("token", token);
      token.add("value", new String(t.termBuffer(), 0, t.termLength()));
      token.add("start", t.startOffset());
      token.add("end", t.endOffset());
      token.add("posInc", t.getPositionIncrement());
      token.add("type", t.type());
      //TODO: handle payloads
    }
    return tokens;
  }
View Full Code Here
160
161
162
163
164
165
166
167
168
169
170
        boolean isCJ = false;
        try {
            Token t = new Token();
            while ((t = ts.next(t)) != null) {
                count++;
                isCJ = StandardTokenizer.TOKEN_TYPES[StandardTokenizer.CJ].equals(t.type());
            }
        } catch (IOException e) {
            throw new ParseException(e.getMessage());
        } finally {
            try {
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
    if (t == null)
      return null;

    char[] buffer = t.termBuffer();
    final int bufferLength = t.termLength();
    final String type = t.type();

    if (type == APOSTROPHE_TYPE &&      // remove 's
  bufferLength >= 2 &&
        buffer[bufferLength-2] == '\'' &&
        (buffer[bufferLength-1] == 's' || buffer[bufferLength-1] == 'S')) {
View Full Code Here
324
325
326
327
328
329
330
331
332
333
334
            final List<Token> expected = actualTokens.get(term).get(docIDToID.get(doc));
            //System.out.println("      doc=" + docIDToID.get(doc) + " " + expected.size() + " freq");
            assertNotNull(expected);
            assertEquals(expected.size(), docsAndPositions.freq());
            for(Token token : expected) {
              int pos = Integer.parseInt(token.type());
              //System.out.println("        pos=" + pos);
              assertEquals(pos, docsAndPositions.nextPosition());
            }
          }
View Full Code Here
339
340
341
342
343
344
345
346
347
348
349
            final List<Token> expected = actualTokens.get(term).get(docIDToID.get(doc));
            //System.out.println("      doc=" + docIDToID.get(doc) + " " + expected.size() + " freq");
            assertNotNull(expected);
            assertEquals(expected.size(), docsAndPositionsAndOffsets.freq());
            for(Token token : expected) {
              int pos = Integer.parseInt(token.type());
              //System.out.println("        pos=" + pos);
              assertEquals(pos, docsAndPositionsAndOffsets.nextPosition());
              assertEquals(token.startOffset(), docsAndPositionsAndOffsets.startOffset());
              assertEquals(token.endOffset(), docsAndPositionsAndOffsets.endOffset());
            }
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
        Token token = tokens[index++];
        termAtt.setEmpty().append(token);
        offsetAtt.setOffset(token.startOffset(), token.endOffset());
        posIncAtt.setPositionIncrement(token.getPositionIncrement());
        flagsAtt.setFlags(token.getFlags());
        typeAtt.setType(token.type());
        payloadAtt.setPayload(token.getPayload());
        return true;
      }
    }
  }
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
    if (tk == null) {
      return null;
    }
    String text = tk.termText();
    if (UnicodeBlock.of(text.charAt(0)) != UnicodeBlock.THAI) {
      return new Token(text.toLowerCase(), tk.startOffset(), tk.endOffset(), tk.type());
    }
    thaiToken = tk;
    breaker.setText(text);
    int end = breaker.next();
    if (end != BreakIterator.DONE) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.