Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.MockAnalyzer.tokenStream()

172
173
174
175
176
177
178
179
180
181
182
  public void testEndOffsetPositionWithCachingTokenFilter() throws Exception {
    Directory dir = newDirectory();
    Analyzer analyzer = new MockAnalyzer(random());
    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer));
    Document doc = new Document();
    TokenStream stream = analyzer.tokenStream("field", "abcd   ");
    stream.reset(); // TODO: weird to reset before wrapping with CachingTokenFilter... correct?
    stream = new CachingTokenFilter(stream);
    FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
    customType.setStoreTermVectors(true);
    customType.setStoreTermVectorPositions(true);
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
    while(true) {
      String s = _TestUtil.randomRealisticUnicodeString(random());
      if (other != null && s.equals(other)) {
        continue;
      }
      final TokenStream ts = a.tokenStream("foo", s);
      final TermToBytesRefAttribute termAtt = ts.getAttribute(TermToBytesRefAttribute.class);
      final BytesRef termBytes = termAtt.getBytesRef();
      ts.reset();

      int count = 0;
View Full Code Here
174
175
176
177
178
179
180
181
182
183
184
  public void testEndOffsetPositionWithCachingTokenFilter() throws Exception {
    Directory dir = newDirectory();
    Analyzer analyzer = new MockAnalyzer(random());
    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer));
    Document doc = new Document();
    TokenStream stream = analyzer.tokenStream("field", new StringReader("abcd   "));
    stream.reset(); // TODO: weird to reset before wrapping with CachingTokenFilter... correct?
    stream = new CachingTokenFilter(stream);
    FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
    customType.setStoreTermVectors(true);
    customType.setStoreTermVectorPositions(true);
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
    while(true) {
      String s = _TestUtil.randomRealisticUnicodeString(random());
      if (other != null && s.equals(other)) {
        continue;
      }
      final TokenStream ts = a.tokenStream("foo", new StringReader(s));
      final TermToBytesRefAttribute termAtt = ts.getAttribute(TermToBytesRefAttribute.class);
      final BytesRef termBytes = termAtt.getBytesRef();
      ts.reset();

      int count = 0;
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
      String s = TestUtil.randomRealisticUnicodeString(random());
      if (other != null && s.equals(other)) {
        continue;
      }
      IOException priorException = null;
      TokenStream ts = a.tokenStream("foo", s);
      try {
        final TermToBytesRefAttribute termAtt = ts.getAttribute(TermToBytesRefAttribute.class);
        final BytesRef termBytes = termAtt.getBytesRef();
        ts.reset();
View Full Code Here
174
175
176
177
178
179
180
181
182
183
184
    Directory dir = newDirectory();
    Analyzer analyzer = new MockAnalyzer(random());
    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer));
    Document doc = new Document();
    IOException priorException = null;
    TokenStream stream = analyzer.tokenStream("field", "abcd   ");
    try {
      stream.reset(); // TODO: weird to reset before wrapping with CachingTokenFilter... correct?
      TokenStream cachedStream = new CachingTokenFilter(stream);
      FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
      customType.setStoreTermVectors(true);
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
        return new TokenStreamComponents(tokenizer, tokenizer);
      }
    };
    for (int i = 0; i < 1000; i++) {
      String s = TestUtil.randomSimpleString(random);
      assertEquals(s, left.tokenStream("foo", newStringReader(s)),
                   right.tokenStream("foo", newStringReader(s)));
    }
  }
 
  // not so useful since its all one token?!
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
      }
    };
    int numIterations = atLeast(50);
    for (int i = 0; i < numIterations; i++) {
      String s = TestUtil.randomSimpleString(random, maxLength);
      assertEquals(s, left.tokenStream("foo", newStringReader(s)),
                   right.tokenStream("foo", newStringReader(s)));
    }
  }
 
  public void testLetterHtmlish() throws Exception {
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
        return new TokenStreamComponents(tokenizer, tokenizer);
      }
    };
    for (int i = 0; i < 1000; i++) {
      String s = TestUtil.randomHtmlishString(random, 20);
      assertEquals(s, left.tokenStream("foo", newStringReader(s)),
                   right.tokenStream("foo", newStringReader(s)));
    }
  }
 
  public void testLetterHtmlishHuge() throws Exception {
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
      }
    };
    int numIterations = atLeast(50);
    for (int i = 0; i < numIterations; i++) {
      String s = TestUtil.randomHtmlishString(random, maxLength);
      assertEquals(s, left.tokenStream("foo", newStringReader(s)),
                   right.tokenStream("foo", newStringReader(s)));
    }
  }
 
  public void testLetterUnicode() throws Exception {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.