Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Analyzer.tokenStream()

67
68
69
70
71
72
73
74
75
76
77
     */
    public TokenStream tokenStream(String fieldName, Reader reader) {
        if (indexingConfig != null) {
            Analyzer propertyAnalyzer = indexingConfig.getPropertyAnalyzer(fieldName);
            if (propertyAnalyzer != null) {
                return propertyAnalyzer.tokenStream(fieldName, reader);
            }
        }
        return defaultAnalyzer.tokenStream(fieldName, reader);
    }

View Full Code Here
104
105
106
107
108
109
110
111
    if ("anchor".equals(fieldName))
      analyzer = ANCHOR_ANALYZER;
    else
      analyzer = CONTENT_ANALYZER;

    return analyzer.tokenStream(fieldName, reader);
  }
}
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
     */
    public TokenStream tokenStream(String fieldName, Reader reader) {
        if (indexingConfig != null) {
            Analyzer propertyAnalyzer = indexingConfig.getPropertyAnalyzer(fieldName);
            if (propertyAnalyzer != null) {
                return propertyAnalyzer.tokenStream(fieldName, reader);
            }
        }
        return defaultAnalyzer.tokenStream(fieldName, reader);
    }

View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
      }
    };
    for (int i = 0; i < 1000; i++) {
      String s = _TestUtil.randomSimpleString(random);
      assertEquals(s, left.tokenStream("foo", newStringReader(s)),
                   right.tokenStream("foo", newStringReader(s)));
    }
  }
 
  // not so useful since its all one token?!
  public void testLetterAsciiHuge() throws Exception {
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
    };
    int numIterations = atLeast(50);
    for (int i = 0; i < numIterations; i++) {
      String s = _TestUtil.randomSimpleString(random, maxLength);
      assertEquals(s, left.tokenStream("foo", newStringReader(s)),
                   right.tokenStream("foo", newStringReader(s)));
    }
  }
 
  public void testLetterHtmlish() throws Exception {
    Random random = random();
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
      }
    };
    for (int i = 0; i < 1000; i++) {
      String s = _TestUtil.randomHtmlishString(random, 20);
      assertEquals(s, left.tokenStream("foo", newStringReader(s)),
                   right.tokenStream("foo", newStringReader(s)));
    }
  }
 
  public void testLetterHtmlishHuge() throws Exception {
    Random random = random();
View Full Code Here
134
135
136
137
138
139
140
141
142
143
144
    };
    int numIterations = atLeast(50);
    for (int i = 0; i < numIterations; i++) {
      String s = _TestUtil.randomHtmlishString(random, maxLength);
      assertEquals(s, left.tokenStream("foo", newStringReader(s)),
                   right.tokenStream("foo", newStringReader(s)));
    }
  }
 
  public void testLetterUnicode() throws Exception {
    Random random = random();
View Full Code Here
151
152
153
154
155
156
157
158
159
160
161
      }
    };
    for (int i = 0; i < 1000; i++) {
      String s = _TestUtil.randomUnicodeString(random);
      assertEquals(s, left.tokenStream("foo", newStringReader(s)),
                   right.tokenStream("foo", newStringReader(s)));
    }
  }
 
  public void testLetterUnicodeHuge() throws Exception {
    Random random = random();
View Full Code Here
171
172
173
174
175
176
177
178
179
180
181
    };
    int numIterations = atLeast(50);
    for (int i = 0; i < numIterations; i++) {
      String s = _TestUtil.randomUnicodeString(random, maxLength);
      assertEquals(s, left.tokenStream("foo", newStringReader(s)),
                   right.tokenStream("foo", newStringReader(s)));
    }
  }
 
  // we only check a few core attributes here.
  // TODO: test other things
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
      // otherwise we can't
      mock.setEnableChecks(consumeAll);
      Analyzer a = new LimitTokenCountAnalyzer(mock, 2, consumeAll);
   
      // dont use assertAnalyzesTo here, as the end offset is not the end of the string (unless consumeAll is true, in which case its correct)!
      assertTokenStreamContents(a.tokenStream("dummy", "1  2     3  4  5"), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 16 : null);
      assertTokenStreamContents(a.tokenStream("dummy", "1 2 3 4 5"), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, consumeAll ? 9 : null);
     
      // less than the limit, ensure we behave correctly
      assertTokenStreamContents(a.tokenStream("dummy", "1  "), new String[] { "1" }, new int[] { 0 }, new int[] { 1 }, consumeAll ? 3 : null);
   
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.