Examples of org.apache.lucene.analysis.PorterStemFilter

org.apache.lucene.analysis.PorterStemFilter
Transforms the token stream as per the Porter stemming algorithm. Note: the input to the stemming filter must already be in lower case, so you will need to use LowerCaseFilter or LowerCaseTokenizer farther down the Tokenizer chain in order for this to work properly!
To use this filter with other analyzers, you'll want to write an Analyzer class that sets up the TokenStream chain as you want it. To use this with LowerCaseTokenizer, for example, you'd write an analyzer like this:
```
 class MyAnalyzer extends Analyzer { public final TokenStream tokenStream(String fieldName, Reader reader) { return new PorterStemFilter(new LowerCaseTokenizer(reader)); } } 
```

    TokenStream result = new SentenceTokenizer(reader);
    result = new WordTokenFilter(result);
    // result = new LowerCaseFilter(result);
    // LowerCaseFilter is not needed, as SegTokenFilter lowercases Basic Latin text.
    // The porter stemming is too strict, this is not a bug, this is a feature:)
    result = new PorterStemFilter(result);
    if (stopWords != null) {
      result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                              result, stopWords, false);
    }
    return result;

View Full Code Here

    if (streams == null) {
      streams = new SavedStreams();
      setPreviousTokenStream(streams);
      streams.tokenStream = new SentenceTokenizer(reader);
      streams.filteredTokenStream = new WordTokenFilter(streams.tokenStream);
      streams.filteredTokenStream = new PorterStemFilter(streams.filteredTokenStream);
      if (stopWords != null) {
        streams.filteredTokenStream = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                                                     streams.filteredTokenStream, stopWords, false);
      }
    } else {

View Full Code Here

    TokenStream result = new SentenceTokenizer(reader);
    result = new WordTokenFilter(result);
    // result = new LowerCaseFilter(result);
    // LowerCaseFilter is not needed, as SegTokenFilter lowercases Basic Latin text.
    // The porter stemming is too strict, this is not a bug, this is a feature:)
    result = new PorterStemFilter(result);
    if (!stopWords.isEmpty()) {
      result = new StopFilter(matchVersion, result, stopWords, false);
    }
    return result;
  }

View Full Code Here

    if (streams == null) {
      streams = new SavedStreams();
      setPreviousTokenStream(streams);
      streams.tokenStream = new SentenceTokenizer(reader);
      streams.filteredTokenStream = new WordTokenFilter(streams.tokenStream);
      streams.filteredTokenStream = new PorterStemFilter(streams.filteredTokenStream);
      if (!stopWords.isEmpty()) {
        streams.filteredTokenStream = new StopFilter(matchVersion, streams.filteredTokenStream, stopWords, false);
      }
    } else {
      streams.tokenStream.reset(reader);

View Full Code Here

    TokenStream result = new StandardFilter(LUCENE_VERSION, tokenizer);
    result = new LowerCaseFilter(LUCENE_VERSION, result);
    result = new ASCIIFoldingFilter(result);
    result = new AlphaNumericMaxLengthFilter(result);
    result = new StopFilter(LUCENE_VERSION, result, stopwords);
    result = new PorterStemFilter(result);
    return new TokenStreamComponents(tokenizer, result);
  }

View Full Code Here

        result = new LowerCaseFilter(matchVersion, result);
        result = new StopFilter(matchVersion, result, stopwords);
        if (stem) {
            if (!stemExclusionSet.isEmpty())
                result = new KeywordMarkerFilter(result, stemExclusionSet);
            result = new PorterStemFilter(result);
            result = new StopFilter(matchVersion, result, stopwords);
        }
        return new TokenStreamComponents(source, result);
    }

View Full Code Here


        if (stem) {
            if (!stemExclusionSet.isEmpty()) {
                result = new KeywordMarkerFilter(result, stemExclusionSet);
            }
            result = new PorterStemFilter(result);
        }


        return new TokenStreamComponents(source, result);
    }

View Full Code Here


        if (stem) {
            if (!stemExclusionSet.isEmpty()) {
                result = new KeywordMarkerFilter(result, stemExclusionSet);
            }
            result = new PorterStemFilter(result);
        }


        return new TokenStreamComponents(source, result);
    }

View Full Code Here

        result = new LowerCaseFilter(matchVersion, result);
        result = new StopFilter(matchVersion, result, stopwords);
        if (stem) {
            if (!stemExclusionSet.isEmpty())
                result = new KeywordMarkerFilter(result, stemExclusionSet);
            result = new PorterStemFilter(result);
            result = new StopFilter(matchVersion, result, stopwords);
        }
        return new TokenStreamComponents(source, result);
    }

View Full Code Here

      result = new EnglishPossessiveFilter(result);
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(!stemExclusionSet.isEmpty())
      result = new KeywordMarkerFilter(result, stemExclusionSet);
    result = new PorterStemFilter(result);
    return new TokenStreamComponents(source, result);
  }

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.lucene.analysis.PorterStemFilter

BugAnalyzer.PorterStemAnalyzer

com.mozilla.grouperfish.lucene.analysis.en.EnglishAnalyzer

com.mozilla.grouperfish.lucene.analysis.en.NGramEnglishAnalyzer

com.mozilla.grouperfish.transforms.coclustering.lucene.analysis.en.EnglishAnalyzer

com.mozilla.grouperfish.transforms.coclustering.lucene.analysis.en.NGramEnglishAnalyzer

edu.wiki.index.WikipediaAnalyzer

ivory.core.tokenize.LuceneTokenizer

mia.clustering.ch10.MyAnalyzer

mia.clustering.ch12.TwitterAnalyzer

org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.