Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.KeywordMarkerFilter

123
124
125
126
127
128
129
130
131
    final Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new StandardFilter(matchVersion, source);
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(!stemExclusionSet.isEmpty())
      result = new KeywordMarkerFilter(result, stemExclusionSet);
    result = new SnowballFilter(result, new FinnishStemmer());
    return new TokenStreamComponents(source, result);
  }
View Full Code Here
123
124
125
126
127
128
129
130
131
    final Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new StandardFilter(matchVersion, source);
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(!stemExclusionSet.isEmpty())
      result = new KeywordMarkerFilter(result, stemExclusionSet);
    result = new SnowballFilter(result, new SwedishStemmer());
    return new TokenStreamComponents(source, result);
  }
View Full Code Here
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
      TokenStream result = new StandardFilter(matchVersion, source);
      result = new ElisionFilter(matchVersion, result);
      result = new LowerCaseFilter(matchVersion, result);
      result = new StopFilter(matchVersion, result, stopwords);
      if(!excltable.isEmpty())
        result = new KeywordMarkerFilter(result, excltable);
      result = new SnowballFilter(result, new org.tartarus.snowball.ext.FrenchStemmer());
      return new TokenStreamComponents(source, result);
    } else {
      final Tokenizer source = new StandardTokenizer(matchVersion, reader);
      TokenStream result = new StandardFilter(matchVersion, source);
      result = new StopFilter(matchVersion, result, stopwords);
      if(!excltable.isEmpty())
        result = new KeywordMarkerFilter(result, excltable);
      result = new FrenchStemFilter(result);
      // Convert to lowercase after stemming!
      return new TokenStreamComponents(source, new LowerCaseFilter(matchVersion, result));
    }
  }
View Full Code Here
220
221
222
223
224
225
226
227
    Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new LowerCaseFilter(matchVersion, source);
    result = new StandardFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(excltable != null && !excltable.isEmpty())
      result = new KeywordMarkerFilter(result, excltable);
    return new TokenStreamComponents(source, new BrazilianStemFilter(result));
  }
View Full Code Here
134
135
136
137
138
139
140
141
142
    final Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new StandardFilter(matchVersion, source);
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(!stemExclusionSet.isEmpty())
      result = new KeywordMarkerFilter(result, stemExclusionSet);
    result = new BulgarianStemFilter(result);
    return new TokenStreamComponents(source, result);
  }
View Full Code Here
123
124
125
126
127
128
129
130
131
    final Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new StandardFilter(matchVersion, source);
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(!stemExclusionSet.isEmpty())
      result = new KeywordMarkerFilter(result, stemExclusionSet);
    result = new SnowballFilter(result, new PortugueseStemmer());
    return new TokenStreamComponents(source, result);
  }
View Full Code Here
121
122
123
124
125
126
127
128
129
    final Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new StandardFilter(matchVersion, source);
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(!stemExclusionSet.isEmpty())
      result = new KeywordMarkerFilter(result, stemExclusionSet);
    result = new GalicianStemFilter(result);
    return new TokenStreamComponents(source, result);
  }
View Full Code Here
128
129
130
131
132
133
134
135
136
137
138
        }
        result = new LowerCaseFilter(matchVersion, result);
        result = new StopFilter(matchVersion, result, stopwords);
        if (stem) {
            if (!stemExclusionSet.isEmpty())
                result = new KeywordMarkerFilter(result, stemExclusionSet);
            result = new PorterStemFilter(result);
            result = new StopFilter(matchVersion, result, stopwords);
        }
        return new TokenStreamComponents(source, result);
    }
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
        }
        result = sf;

        if (stem) {
            if (!stemExclusionSet.isEmpty()) {
                result = new KeywordMarkerFilter(result, stemExclusionSet);
            }
            result = new PorterStemFilter(result);
        }

        return new TokenStreamComponents(source, result);
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
    } else {
      source = new IndicTokenizer(matchVersion, reader);
    }
    TokenStream result = new LowerCaseFilter(matchVersion, source);
    if (!stemExclusionSet.isEmpty())
      result = new KeywordMarkerFilter(result, stemExclusionSet);
    result = new IndicNormalizationFilter(result);
    result = new HindiNormalizationFilter(result);
    result = new StopFilter(matchVersion, result, stopwords);
    result = new HindiStemFilter(result);
    return new TokenStreamComponents(source, result);
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.KeywordMarkerFilter

  • com.github.pmerienne.trident.ml.preprocessing.EnglishTokenizer$EnglishSpecialAnalyzer
  • com.mozilla.grouperfish.lucene.analysis.en.EnglishAnalyzer
  • com.mozilla.grouperfish.lucene.analysis.en.NGramEnglishAnalyzer
  • com.mozilla.grouperfish.transforms.coclustering.lucene.analysis.en.EnglishAnalyzer
  • com.mozilla.grouperfish.transforms.coclustering.lucene.analysis.en.NGramEnglishAnalyzer
  • org.apache.lucene.analysis.ar.ArabicAnalyzer
  • org.apache.lucene.analysis.ar.TestArabicStemFilter
  • org.apache.lucene.analysis.bg.BulgarianAnalyzer
  • org.apache.lucene.analysis.bg.TestBulgarianStemmer
  • org.apache.lucene.analysis.br.BrazilianAnalyzer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.