Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
        int maxNumFragmentsRequired = 3;

        for (int i = 0; i < hits.totalHits; i++) {
          String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream, HighlighterTest.this, false);

          highlighter.setTextFragmenter(new SimpleFragmenter(40));

          String highlightedText = highlighter.getBestFragments(tokenStream, text,
View Full Code Here
151
152
153
154
155
156
157
158
159
160
161
    Analyzer analyzer = new ShingleAnalyzerWrapper(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false), 2);
    searcher = setUpSearcher(analyzer);

    PhraseQuery q = new PhraseQuery();

    TokenStream ts = analyzer.tokenStream("content",
                                          new StringReader("this sentence"));
    int j = -1;
   
    PositionIncrementAttribute posIncrAtt = ts.addAttribute(PositionIncrementAttribute.class);
    CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
    Analyzer analyzer = new ShingleAnalyzerWrapper(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false), 2);
    searcher = setUpSearcher(analyzer);

    BooleanQuery q = new BooleanQuery();

    TokenStream ts = analyzer.tokenStream("content",
                                          new StringReader("test sentence"));
   
    CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
   
    ts.reset();
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
     */
    protected Query getPrefixQuery(String field, String termStr)
            throws ParseException {
        // only create a prefix query when the term is a single word / token
        Analyzer a = getAnalyzer();
        TokenStream ts = a.tokenStream(field, new StringReader(termStr));
        int count = 0;
        boolean isCJ = false;
        try {
            Token t = new Token();
            while ((t = ts.next(t)) != null) {
View Full Code Here
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
        // retrieve analyzer
        Analyzer analyzer = retrieveAnalyzer(content);
               
        // @see org.apache.lucene.analysis.Analyzer - null for fieldname should be supported
        // most analyzers do not use the parameter fieldname
        TokenStream tokenStream = analyzer.tokenStream(null, new StringReader(text));
        return tokenStream;
    }
   
    /**
     *
 
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
    while(true) {
      String s = TestUtil.randomRealisticUnicodeString(random());
      if (other != null && s.equals(other)) {
        continue;
      }
      try (TokenStream ts = a.tokenStream("foo", s)) {
        final TermToBytesRefAttribute termAtt = ts.getAttribute(TermToBytesRefAttribute.class);
        final BytesRef termBytes = termAtt.getBytesRef();
        ts.reset();

        int count = 0;
View Full Code Here
176
177
178
179
180
181
182
183
184
185
186
  public void testEndOffsetPositionWithCachingTokenFilter() throws Exception {
    Directory dir = newDirectory();
    Analyzer analyzer = new MockAnalyzer(random());
    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer));
    Document doc = new Document();
    try (TokenStream stream = analyzer.tokenStream("field", "abcd   ")) {
      stream.reset(); // TODO: weird to reset before wrapping with CachingTokenFilter... correct?
      TokenStream cachedStream = new CachingTokenFilter(stream);
      FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
      customType.setStoreTermVectors(true);
      customType.setStoreTermVectorPositions(true);
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
      }
    };
    for (int i = 0; i < 1000; i++) {
      String s = TestUtil.randomSimpleString(random);
      assertEquals(s, left.tokenStream("foo", newStringReader(s)),
                   right.tokenStream("foo", newStringReader(s)));
    }
  }
 
  // not so useful since its all one token?!
  public void testLetterAsciiHuge() throws Exception {
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
    };
    int numIterations = atLeast(50);
    for (int i = 0; i < numIterations; i++) {
      String s = TestUtil.randomSimpleString(random, maxLength);
      assertEquals(s, left.tokenStream("foo", newStringReader(s)),
                   right.tokenStream("foo", newStringReader(s)));
    }
  }
 
  public void testLetterHtmlish() throws Exception {
    Random random = random();
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
      }
    };
    for (int i = 0; i < 1000; i++) {
      String s = TestUtil.randomHtmlishString(random, 20);
      assertEquals(s, left.tokenStream("foo", newStringReader(s)),
                   right.tokenStream("foo", newStringReader(s)));
    }
  }
 
  public void testLetterHtmlishHuge() throws Exception {
    Random random = random();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.