Examples of org.apache.lucene.analysis.StopFilter

org.apache.lucene.analysis.StopFilter
Removes stop words from a token stream.

     */
    public TokenStream tokenStream(String fieldName, Reader reader)
    {
        TokenStream result = new RussianLetterTokenizer(reader, charset);
        result = new RussianLowerCaseFilter(result, charset);
        result = new StopFilter(result, stoptable);
        result = new RussianStemFilter(result, charset);
        return result;
    }

View Full Code Here

    public TokenStream tokenStream( String fieldName, Reader reader )
    {
  TokenStream result = new StandardTokenizer( reader );
  result = new StandardFilter( result );
  // shouldn't there be a lowercaser before stop word filtering?
  result = new StopFilter( result, stoptable );
  result = new GermanStemFilter( result, excltable );
  return result;
    }

View Full Code Here

    } catch (IOException e) {
      result = delegate.tokenStream(fieldName, reader);
    }
    HashSet<String> stopWords = stopWordsPerField.get(fieldName);
    if (stopWords != null) {
      result = new StopFilter(matchVersion, result, stopWords);
    }
    return result;
  }

View Full Code Here

      streams.wrapped = delegate.reusableTokenStream(fieldName, reader);


      /* if there are any stopwords for the field, save the stopfilter */
      HashSet<String> stopWords = stopWordsPerField.get(fieldName);
      if (stopWords != null)
        streams.withStopFilter = new StopFilter(matchVersion, streams.wrapped, stopWords);
      else
        streams.withStopFilter = streams.wrapped;


    } else {
      /*
       * an entry for this field exists, verify the wrapped stream has not
       * changed. if it has not, reuse it, otherwise wrap the new stream.
       */
      TokenStream result = delegate.reusableTokenStream(fieldName, reader);
      if (result == streams.wrapped) {
        /* the wrapped analyzer reused the stream */
      } else {
        /*
         * the wrapped analyzer did not. if there are any stopwords for the
         * field, create a new StopFilter around the new stream
         */
        streams.wrapped = result;
        HashSet<String> stopWords = stopWordsPerField.get(fieldName);
        if (stopWords != null)
          streams.withStopFilter = new StopFilter(matchVersion, streams.wrapped, stopWords);
        else
          streams.withStopFilter = streams.wrapped;
      }
    }

View Full Code Here

  protected TokenStreamComponents createComponents(String fieldName,
      Reader reader) {
    Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new LowerCaseFilter(matchVersion, source);
    result = new StandardFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(excltable != null && !excltable.isEmpty())
      result = new KeywordMarkerFilter(result, excltable);
    return new TokenStreamComponents(source, new BrazilianStemFilter(result));
  }

View Full Code Here

    TokenStream result = new StandardFilter(matchVersion, source);
    if (matchVersion.onOrAfter(Version.LUCENE_32)) {
      result = new ElisionFilter(matchVersion, result, DEFAULT_ARTICLES);
    }
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(!stemExclusionSet.isEmpty())
      result = new KeywordMarkerFilter(result, stemExclusionSet);
    result = new SnowballFilter(result, new ItalianStemmer());
    return new TokenStreamComponents(source, result);
  }

View Full Code Here

     * StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}.
     */
    public TokenStream tokenStream(String fieldName, Reader reader) {
        TokenStream result = new WhitespaceTokenizer(reader);
        result = new StandardFilter(result);
        result = new StopFilter(result, stopSet);
        return result;
    }

View Full Code Here

  public Set getStopWords() {
    return stopWords;
  }


  public StopFilter create(TokenStream input) {
    StopFilter stopFilter = new StopFilter(input,stopWords,ignoreCase);
    stopFilter.setEnablePositionIncrements(enablePositionIncrements);
    return stopFilter;
  }

View Full Code Here

        this.ignoreCase = settings.getAsBoolean("ignore_case", false);
        this.enablePositionIncrements = settings.getAsBoolean("enable_position_increments", version.onOrAfter(Version.LUCENE_29));
    }


    @Override public TokenStream create(TokenStream tokenStream) {
        StopFilter filter = new StopFilter(version, tokenStream, stopWords, ignoreCase);
        filter.setEnablePositionIncrements(enablePositionIncrements);
        return filter;
    }

View Full Code Here

     */
    private class StemmingAnalyzer extends Analyzer {
        @Override
    public final TokenStream tokenStream(String fieldName, Reader reader) {
            // Apply stop words and porter stemmer using a lower-case tokenizer.
            TokenStream stream = new StopFilter(new LowerCaseTokenizer(reader),
                StandardAnalyzer.STOP_WORDS);
            return new PorterStemFilter(stream);
        }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.lucene.analysis.StopFilter

BugAnalyzer.PorterStemAnalyzer

com.gentics.cr.lucene.analysis.CustomPatternAnalyzer

com.gentics.cr.lucene.autocomplete.AutocompleteAnalyzer

com.github.pmerienne.trident.ml.preprocessing.EnglishTokenizer$EnglishSpecialAnalyzer

com.infoclinika.mssharing.search.NoStopWordsAnalyzer

com.mozilla.grouperfish.lucene.analysis.en.EnglishAnalyzer

com.mozilla.grouperfish.transforms.coclustering.lucene.analysis.en.EnglishAnalyzer

edu.wiki.index.WikipediaAnalyzer

ivory.core.tokenize.LuceneArabicAnalyzer

ivory.core.tokenize.LuceneSpanishAnalyzer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.