Package org.apache.lucene.analysis.standard

Examples of org.apache.lucene.analysis.standard.StandardAnalyzer.tokenStream()

485
486
487
488
489
490
491
492
493
494
495
  }
 
  protected Set<String> getHighlightWords(String searchString) {
    try {
      Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT);
      TokenStream stream = analyzer.tokenStream("content", new StringReader(searchString));
      TermAttribute termAtt = (TermAttribute) stream.addAttribute(TermAttribute.class);
      for (boolean next = stream.incrementToken(); next; next = stream.incrementToken()) {
        String term = termAtt.term();
        if(log.isDebug()) log.debug(term);
      }
View Full Code Here
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
        int maxNumFragmentsRequired = 3;

        for (int i = 0; i < hits.totalHits; i++) {
          String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream, HighlighterTest.this, false);

          highlighter.setTextFragmenter(new SimpleFragmenter(40));

          String highlightedText = highlighter.getBestFragments(tokenStream, text,
View Full Code Here
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
        int maxNumFragmentsRequired = 3;

        for (int i = 0; i < hits.totalHits; i++) {
          String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream, HighlighterTest.this, false);

          highlighter.setTextFragmenter(new SimpleFragmenter(40));

          String highlightedText = highlighter.getBestFragments(tokenStream, text,
View Full Code Here
23
24
25
26
27
28
29
30
31
32
33
  public static void main(String[] args) throws IOException {
    FeatureVectorEncoder encoder = new StaticWordValueEncoder("text");
    Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_31);    

    StringReader in = new StringReader("text to magically vectorize");
    TokenStream ts = analyzer.tokenStream("body", in);
    TermAttribute termAtt = ts.addAttribute(TermAttribute.class);

    Vector v1 = new RandomAccessSparseVector(100);                  
    while (ts.incrementToken()) {
      char[] termBuffer = termAtt.termBuffer();
View Full Code Here
182
183
184
185
186
187
188
189
190
191
192
              String superColumnName = merge(superColumnList, " ");
              superColumnList.clear();
              if(importer_.columnFamily.superColumn.tokenize)
              {
                  Analyzer analyzer = new StandardAnalyzer();
                  TokenStream ts = analyzer.tokenStream("superColumn", new StringReader(superColumnName));
                  Token token = null;
                  token = ts.next();
                  while(token != null)
                  {
                    superColumnList.add(token.termText());
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
      ClassifierContext ctx = new ClassifierContext(a,ds);
      ctx.initialize();
     
      //TODO: make the analyzer configurable
      StandardAnalyzer analyzer = new StandardAnalyzer(Version.LUCENE_36);
      TokenStream ts = analyzer.tokenStream(null, new InputStreamReader(new FileInputStream(inputFile), "UTF-8"));
    
      ArrayList<String> tokens = new ArrayList<String>(1000);
      while (ts.incrementToken()) {
        tokens.add(ts.getAttribute(CharTermAttribute.class).toString());
      }
View Full Code Here
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
        int maxNumFragmentsRequired = 3;

        for (int i = 0; i < hits.length(); i++) {
          String text = hits.doc(i).get(FIELD_NAME);
          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream, HighlighterTest.this, false);

          highlighter.setTextFragmenter(new SimpleFragmenter(40));

          String highlightedText = highlighter.getBestFragments(tokenStream, text,
View Full Code Here
943
944
945
946
947
948
949
950
951
952
953
  private List<String> tokenize(String string)
  {
    List<String> stringList = new ArrayList<String>();
      Analyzer analyzer = new StandardAnalyzer();
      TokenStream ts = analyzer.tokenStream("superColumn", new StringReader(string));
      Token token = null;
      try
      {
        token = ts.next();
        while(token != null)
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
        // TODO: Decouple from lucene, allow the analyzer to be configurable.
        // TODO: Verifiy that it is necessary to create a new analyzer instance each time.
        Analyzer analyzer = new StandardAnalyzer();
        Reader docReader = new StringReader(doc);
        TokenStream tokenStream = analyzer.tokenStream(null, docReader);
       
        try {
            if (1 < maxTuple ){
                tokenStream= new TupleTokenizer(tokenStream, maxTuple);
            }
View Full Code Here
389
390
391
392
393
394
395
396
397
398
399
     QueryScorer qScorer = new QueryScorer(query);
     SimpleHTMLFormatter formatter = new SimpleHTMLFormatter("<span class=\"hlight\">", "</span>");
     Highlighter highlighter = new Highlighter(formatter, qScorer);
     Fragmenter fragmenter = new SimpleFragmenter(80);        //*-- use fragments of 50 bytes each
     highlighter.setTextFragmenter(fragmenter);
     TokenStream tokenStream = sAnalyzer.tokenStream("contents", new StringReader(text));
     result = highlighter.getBestFragments(tokenStream, text, 3, "...<br>");  //*-- collect upto three fragments
   }
  }
  catch (ParseException pe) { logger.error("Query parse error " + pe.getMessage() ); }

View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.