Package org.apache.lucene.analysis.standard

Examples of org.apache.lucene.analysis.standard.StandardFilter

114
115
116
117
118
119
120
121
122
123
124
  }
 
  @Override
  protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
    Tokenizer tokenizer = new StandardTokenizer(Version.LUCENE_31, reader);
    TokenStream result = new StandardFilter(Version.LUCENE_31, tokenizer);
    result = new LowerCaseFilter(Version.LUCENE_31, result);
    result = new ASCIIFoldingFilter(result);
    result = new AlphaNumericMaxLengthFilter(result);
    result = new StopFilter(Version.LUCENE_31, result, stopwords);
    result = new PorterStemFilter(result);
View Full Code Here
42
43
44
45
46
47
48
49
50
51
  }
 
  @Override
  protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
    Tokenizer tokenizer = new WikipediaTokenizer(reader);
    TokenStream result = new StandardFilter(Version.LUCENE_31, tokenizer);
    result = new LowerCaseFilter(Version.LUCENE_31, result);
    result = new StopFilter(Version.LUCENE_31, result, stopwords);
    return new TokenStreamComponents(tokenizer, result);
  }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
  }
 
  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new WikipediaTokenizer(reader);
    result = new StandardFilter(result);
    result = new LowerCaseFilter(result);
    result = new StopFilter(true, result, stopSet);
    return result;
  }
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
   */
  @Override
  protected TokenStreamComponents createComponents(String fieldName,
      Reader reader) {
    final Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new StandardFilter(matchVersion, source);
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(!stemExclusionSet.isEmpty())
      result = new KeywordMarkerFilter(result, stemExclusionSet);
    result = new LatvianStemFilter(result);
View Full Code Here
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
  @Override
  protected TokenStreamComponents createComponents(String fieldName,
      Reader aReader) {
    if (matchVersion.onOrAfter(Version.LUCENE_31)) {
      final Tokenizer source = new StandardTokenizer(matchVersion, aReader);
      TokenStream result = new StandardFilter(matchVersion, source);
      result = new LowerCaseFilter(matchVersion, result);
      result = new StopFilter(matchVersion, result, stoptable);
      if (!excltable.isEmpty())
        result = new KeywordMarkerFilter(result, excltable);
      if (!stemdict.isEmpty())
        result = new StemmerOverrideFilter(matchVersion, result, stemdict);
      result = new SnowballFilter(result, new org.tartarus.snowball.ext.DutchStemmer());
      return new TokenStreamComponents(source, result);
    } else {
      final Tokenizer source = new StandardTokenizer(matchVersion, aReader);
      TokenStream result = new StandardFilter(matchVersion, source);
      result = new StopFilter(matchVersion, result, stoptable);
      if (!excltable.isEmpty())
        result = new KeywordMarkerFilter(result, excltable);
      result = new DutchStemFilter(result, stemdict);
      return new TokenStreamComponents(source, result);
View Full Code Here
22
23
24
25
26
27
28
29
30
31
32
 
  @Override
  @SuppressWarnings("resource")
  protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
    final StandardTokenizer src = new StandardTokenizer(version, reader);
      TokenStream tok = new StandardFilter(version, src);
      tok = new LowerCaseFilter(version, tok);
      tok = new StopFilter(version, tok, StandardAnalyzer.STOP_WORDS_SET);
      tok = new PorterStemFilter(tok);
      return new TokenStreamComponents(src, tok);
  }
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
  }

  private TokenStream tokenStream(Reader reader) {
    StandardTokenizer tokenStream = new StandardTokenizer(reader, replaceInvalidAcronym);
    tokenStream.setMaxTokenLength(maxTokenLength);
    TokenStream result = new StandardFilter(tokenStream);

    result = new LowerCaseFilter(result);
    result = new StopFilter(result, TERRIER_STOP_WORDS);
    result = new PorterStemFilter(result);
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
   */
  @Override
  protected TokenStreamComponents createComponents(String fieldName,
      Reader reader) {
    final Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new StandardFilter(matchVersion, source);
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if (!stemExclusionSet.isEmpty()) {
      result = new KeywordMarkerFilter(result, stemExclusionSet);
    }
View Full Code Here
151
152
153
154
155
156
157
158
159
160
161
    @Override
    protected TokenStreamComponents createComponents(String fieldName,
        Reader reader) {
      if (matchVersion.onOrAfter(Version.LUCENE_31)) {
        final Tokenizer source = new StandardTokenizer(matchVersion, reader);
        TokenStream result = new StandardFilter(matchVersion, source);
        result = new LowerCaseFilter(matchVersion, result);
        result = new StopFilter(matchVersion, result, stopwords);
        if (!stemExclusionSet.isEmpty()) result = new KeywordMarkerFilter(
            result, stemExclusionSet);
        result = new SnowballFilter(result, new org.tartarus.snowball.ext.RussianStemmer());
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
   */
  @Override
  protected TokenStreamComponents createComponents(String fieldName,
      Reader reader) {
    final Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new StandardFilter(matchVersion, source);
    result = new TurkishLowerCaseFilter(result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(!stemExclusionSet.isEmpty())
      result = new KeywordMarkerFilter(result, stemExclusionSet);
    result = new SnowballFilter(result, new TurkishStemmer());
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.standard.StandardFilter

  • com.foundationdb.server.service.text.SelectiveCaseAnalyzer
  • com.gentics.cr.lucene.autocomplete.AutocompleteAnalyzer
  • com.github.pmerienne.trident.ml.preprocessing.EnglishTokenizer$EnglishSpecialAnalyzer
  • com.googlecode.lucene.PorterAnalyzer
  • com.livingsocial.hive.udf.Tokenize$MyAnalyzer
  • com.mozilla.grouperfish.lucene.analysis.en.EnglishAnalyzer
  • com.mozilla.grouperfish.lucene.analysis.en.NGramEnglishAnalyzer
  • com.mozilla.grouperfish.transforms.coclustering.lucene.analysis.en.EnglishAnalyzer
  • com.mozilla.grouperfish.transforms.coclustering.lucene.analysis.en.NGramEnglishAnalyzer
  • edu.wiki.index.WikipediaAnalyzer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.