Examples of org.apache.lucene.analysis.Tokenizer

org.apache.lucene.analysis.Tokenizer
A Tokenizer is a TokenStream whose input is a Reader.
This is an abstract class.
NOTE: subclasses must override {@link #incrementToken()} if the new TokenStream API is usedand {@link #next(Token)} or {@link #next()} if the oldTokenStream API is used.
NOTE: Subclasses overriding {@link #incrementToken()} mustcall {@link AttributeSource#clearAttributes()} beforesetting attributes. Subclasses overriding {@link #next(Token)} must call{@link Token#clear()} before setting Token attributes.

    final SynonymMap map = b.build();


    final Analyzer analyzer = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
        return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, false));
      }
    };


    assertAnalyzesTo(analyzer, "a b c",

View Full Code Here

      final boolean ignoreCase = random().nextBoolean();
      
      final Analyzer analyzer = new Analyzer() {
        @Override
        protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
          Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
          return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, ignoreCase));
        }
      };


      checkRandomData(random(), analyzer, 100);

View Full Code Here

      final boolean ignoreCase = random.nextBoolean();
      
      final Analyzer analyzer = new Analyzer() {
        @Override
        protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
          Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
          TokenStream syns = new SynonymFilter(tokenizer, map, ignoreCase);
          TokenStream graph = new MockGraphTokenFilter(random(), syns);
          return new TokenStreamComponents(tokenizer, graph);
        }
      };

View Full Code Here

      final boolean ignoreCase = random.nextBoolean();
      
      final Analyzer analyzer = new Analyzer() {
        @Override
        protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
          Tokenizer tokenizer = new KeywordTokenizer(reader);
          return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, ignoreCase));
        }
      };


      checkAnalysisConsistency(random, analyzer, random.nextBoolean(), "");

View Full Code Here

      final boolean ignoreCase = random.nextBoolean();
      
      final Analyzer analyzer = new Analyzer() {
        @Override
        protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
          Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
          return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, ignoreCase));
        }
      };


      checkRandomData(random, analyzer, 100, 1024);

View Full Code Here

    final SynonymMap map = parser.build();
      
    Analyzer analyzer = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
        return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
      }
    };
    
    // where did my pot go?!

View Full Code Here

      final int min = _TestUtil.nextInt(random(), 2, 10);
      final int max = _TestUtil.nextInt(random(), min, 20);
      Analyzer a = new Analyzer() {
        @Override
        protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
          Tokenizer tokenizer = new NGramTokenizer(TEST_VERSION_CURRENT, reader, min, max);
          return new TokenStreamComponents(tokenizer, tokenizer);
        }    
      };
      checkRandomData(random(), a, 200*RANDOM_MULTIPLIER, 20);
      checkRandomData(random(), a, 10*RANDOM_MULTIPLIER, 1027);

View Full Code Here

    add("x c", "xc", keepOrig);
    final SynonymMap map = b.build();
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
      }
    };


    checkOneTerm(a, "$", "$");

View Full Code Here

    add("a b", "ab", keepOrig);
    final SynonymMap map = b.build();
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
      }
    };


    assertAnalyzesTo(a, "a b",

View Full Code Here

    add("a b", "ab", keepOrig);
    final SynonymMap map = b.build();
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
      }
    };


    assertAnalyzesTo(a, "a b",

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.lucene.analysis.Tokenizer

com.github.bibreen.mecab_ko_lucene_analyzer.MeCabKoStandardTokenizerTest

org.apache.lucene.analysis.ar.TestArabicFilters

org.apache.lucene.analysis.ca.CatalanAnalyzer

org.apache.lucene.analysis.charfilter.TestMappingCharFilter

org.apache.lucene.analysis.cjk.TestCJKAnalyzer

org.apache.lucene.analysis.cjk.TestCJKBigramFilter

org.apache.lucene.analysis.cn.smart.TestSmartChineseAnalyzer

org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter

org.apache.lucene.analysis.compound.TestHyphenationCompoundWordTokenFilterFactory

org.apache.lucene.analysis.core.TestBugInSomething

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.