Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.termText()

57
58
59
60
61
62
63
64
65
66
67
      Token token;
      while (encore) {
        token = filter.next();
        encore = token != null;
        if (token != null)
          tas.add(token.termText());
      }
    } catch (IOException e) {
      e.printStackTrace();
    }
    return tas;
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
    TokenStream ts = a.tokenStream("dummy", new StringReader(input));

    for (int i = 0; i < output.length; i++) {
      Token t = ts.next();
      assertNotNull(t);
      assertEquals(t.termText(), output[i]);
    }
    assertNull(ts.next());
    ts.close();
  }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
      Term term = null;
      while (token != null)
      {
        if (term == null)
        {
          term = new Term(fieldName, token.termText());
        } else
        {
//           create from previous to save fieldName.intern overhead
          term = term.createTerm(token.termText());
        }
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
        {
          term = new Term(fieldName, token.termText());
        } else
        {
//           create from previous to save fieldName.intern overhead
          term = term.createTerm(token.termText());
        }
        bq.add(new BooleanClause(new TermQuery(term),BooleanClause.Occur.SHOULD));
        token = ts.next();
      }
    }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
      Term term = null;
      while (token != null)
      {
        if (term == null)
        {
          term = new Term(fieldName, token.termText());
        } else
        {
//           create from previous to save fieldName.intern overhead
          term = term.createTerm(token.termText());
        }
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
        int corpusNumDocs=reader.numDocs();
        Term internSavingTemplateTerm =new Term(f.fieldName,""); //optimization to avoid constructing new Term() objects
        HashSet processedTerms=new HashSet();
        while(token!=null)
        {           
          if(!processedTerms.contains(token.termText()))
          {
            processedTerms.add(token.termText());
                ScoreTermQueue variantsQ=new ScoreTermQueue(MAX_VARIANTS_PER_TERM); //maxNum variants considered for any one term
                float minScore=0;
                Term startTerm=internSavingTemplateTerm.createTerm(token.termText());
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
        HashSet processedTerms=new HashSet();
        while(token!=null)
        {           
          if(!processedTerms.contains(token.termText()))
          {
            processedTerms.add(token.termText());
                ScoreTermQueue variantsQ=new ScoreTermQueue(MAX_VARIANTS_PER_TERM); //maxNum variants considered for any one term
                float minScore=0;
                Term startTerm=internSavingTemplateTerm.createTerm(token.termText());
                FuzzyTermEnum fe=new FuzzyTermEnum(reader,startTerm,f.minSimilarity,f.prefixLength);
                TermEnum origEnum = reader.terms(startTerm);
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
          if(!processedTerms.contains(token.termText()))
          {
            processedTerms.add(token.termText());
                ScoreTermQueue variantsQ=new ScoreTermQueue(MAX_VARIANTS_PER_TERM); //maxNum variants considered for any one term
                float minScore=0;
                Term startTerm=internSavingTemplateTerm.createTerm(token.termText());
                FuzzyTermEnum fe=new FuzzyTermEnum(reader,startTerm,f.minSimilarity,f.prefixLength);
                TermEnum origEnum = reader.terms(startTerm);
                int df=0;
                if(startTerm.equals(origEnum.term()))
                {
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
            assertEquals(
                "Unicode",
                token.termText(),
                sampleToken == null
                ? null
                : sampleToken.termText());
        }

        inWords.close();
        sampleUnicode.close();
    }
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
            assertEquals(
                "KOI8",
                token.termText(),
                sampleToken == null
                ? null
                : sampleToken.termText());

        }

        inWordsKOI8.close();
        sampleKOI8.close();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.