Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

Class org.apache.lucene.analysis.Analyzer

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

org.apache.lucene.analysis.Analyzer.tokenStream()
Creates a TokenStream which tokenizes all the text in the provided Reader. Must be able to handle null field name for backward compatibility.

      mock.setEnableChecks(consumeAll);
      Analyzer a = new LimitTokenCountAnalyzer(mock, 2, consumeAll);
    
      // dont use assertAnalyzesTo here, as the end offset is not the end of the string (unless consumeAll is true, in which case its correct)!
      assertTokenStreamContents(a.tokenStream("dummy", "1  2     3  4  5"), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 16 : null);
      assertTokenStreamContents(a.tokenStream("dummy", "1 2 3 4 5"), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, consumeAll ? 9 : null);
      
      // less than the limit, ensure we behave correctly
      assertTokenStreamContents(a.tokenStream("dummy", "1  "), new String[] { "1" }, new int[] { 0 }, new int[] { 1 }, consumeAll ? 3 : null);
    
      // equal to limit

View Full Code Here

      // dont use assertAnalyzesTo here, as the end offset is not the end of the string (unless consumeAll is true, in which case its correct)!
      assertTokenStreamContents(a.tokenStream("dummy", "1  2     3  4  5"), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 16 : null);
      assertTokenStreamContents(a.tokenStream("dummy", "1 2 3 4 5"), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, consumeAll ? 9 : null);
      
      // less than the limit, ensure we behave correctly
      assertTokenStreamContents(a.tokenStream("dummy", "1  "), new String[] { "1" }, new int[] { 0 }, new int[] { 1 }, consumeAll ? 3 : null);
    
      // equal to limit
      assertTokenStreamContents(a.tokenStream("dummy", "1  2  "), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 6 : null);
    }
  }

View Full Code Here

      
      // less than the limit, ensure we behave correctly
      assertTokenStreamContents(a.tokenStream("dummy", "1  "), new String[] { "1" }, new int[] { 0 }, new int[] { 1 }, consumeAll ? 3 : null);
    
      // equal to limit
      assertTokenStreamContents(a.tokenStream("dummy", "1  2  "), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 6 : null);
    }
  }


  public void testLimitTokenCountIndexWriter() throws IOException {

View Full Code Here

          return new TokenStreamComponents(tokenizer, new LimitTokenPositionFilter(tokenizer, 2, consumeAll));
        }
      };


      // dont use assertAnalyzesTo here, as the end offset is not the end of the string (unless consumeAll is true, in which case its correct)!
      assertTokenStreamContents(a.tokenStream("dummy", "1  2     3  4  5"), 
                                new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 16 : null);
      assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), 
                                new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, consumeAll ? 9 : null);


      // less than the limit, ensure we behave correctly

View Full Code Here

      };


      // dont use assertAnalyzesTo here, as the end offset is not the end of the string (unless consumeAll is true, in which case its correct)!
      assertTokenStreamContents(a.tokenStream("dummy", "1  2     3  4  5"), 
                                new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 16 : null);
      assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), 
                                new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, consumeAll ? 9 : null);


      // less than the limit, ensure we behave correctly
      assertTokenStreamContents(a.tokenStream("dummy", "1  "),
                                new String[] { "1" }, new int[] { 0 }, new int[] { 1 }, consumeAll ? 3 : null);

View Full Code Here

                                new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 16 : null);
      assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), 
                                new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, consumeAll ? 9 : null);


      // less than the limit, ensure we behave correctly
      assertTokenStreamContents(a.tokenStream("dummy", "1  "),
                                new String[] { "1" }, new int[] { 0 }, new int[] { 1 }, consumeAll ? 3 : null);
                                                                                   
      // equal to limit
      assertTokenStreamContents(a.tokenStream("dummy", "1  2  "), 
                                new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 6 : null);

View Full Code Here

      // less than the limit, ensure we behave correctly
      assertTokenStreamContents(a.tokenStream("dummy", "1  "),
                                new String[] { "1" }, new int[] { 0 }, new int[] { 1 }, consumeAll ? 3 : null);
                                                                                   
      // equal to limit
      assertTokenStreamContents(a.tokenStream("dummy", "1  2  "), 
                                new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 6 : null);
    }
  }
  
  public void testMaxPosition3WithSynomyms() throws IOException {

View Full Code Here

      String s = _TestUtil.randomRealisticUnicodeString(random());
      if (other != null && s.equals(other)) {
        continue;
      }
      IOException priorException = null;
      TokenStream ts = a.tokenStream("foo", s);
      try {
        final TermToBytesRefAttribute termAtt = ts.getAttribute(TermToBytesRefAttribute.class);
        final BytesRef termBytes = termAtt.getBytesRef();
        ts.reset();

View Full Code Here

     */
    public TokenStream tokenStream(String fieldName, Reader reader) {
        if (indexingConfig != null) {
            Analyzer propertyAnalyzer = indexingConfig.getPropertyAnalyzer(fieldName);
            if (propertyAnalyzer != null) {
                return propertyAnalyzer.tokenStream(fieldName, reader);
            }
        }
        return defaultAnalyzer.tokenStream(fieldName, reader);
    }

View Full Code Here

      if (leaf != lastLeaf) {
        termsEnum = t.iterator(null);
        postings = new DocsAndPositionsEnum[allTerms.length];
      }
      if (automata.length > 0) {
        DocsAndPositionsEnum dp = MultiTermHighlighting.getDocsEnum(analyzer.tokenStream(field, content), automata);
        dp.advance(doc - subContext.docBase);
        postings[terms.length] = dp;
      }
      Passage passages[] = highlightDoc(field, allTerms, content.length(), bi, doc - subContext.docBase, termsEnum, postings, maxPassages);
      if (passages.length == 0) {

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.