Examples of CannedTokenStream

org.apache.lucene.analysis.CannedTokenStream
TokenStream from a canned list of Tokens.

Examples of org.apache.lucene.analysis.CannedTokenStream


          @Override
          public TokenStream getTokenStream() {
            if (count == 0) {
              count++;
              return new CannedTokenStream(new Token[] {
                  token("p", 1, 1),
                  token("q", 1, 1),
                  token("r", 0, 1),
                  token("s", 0, 1),
                });
            } else {
              return new CannedTokenStream(new Token[] {
                  token("p", 1, 1),
                });
            }
          }

View Full Code Here

Examples of org.apache.lucene.analysis.CannedTokenStream

            public TokenStream getTokenStream() {
              Token a = new Token("a", 0, 1);
              a.setPositionIncrement(1);
              Token b = new Token("b", 0, 1);
              b.setPositionIncrement(0);
              return new CannedTokenStream(new Token[] {a, b});
            }
         
            @Override
            protected void setReader(final Reader reader) throws IOException {
            }

View Full Code Here

Examples of org.apache.lucene.analysis.CannedTokenStream

    Document doc = new Document();
    FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
    ft.setStoreTermVectors(true);
    ft.setStoreTermVectorOffsets(true);
    Field field = new Field("foo", "", ft);
    field.setTokenStream(new CannedTokenStream(
        new Token("bar", 5, 10), new Token("bar", 1, 4)
    ));
    doc.add(field);
    iw.addDocument(doc);
    iw.close();

View Full Code Here

Examples of org.apache.lucene.analysis.CannedTokenStream

    type.setStoreTermVectorPositions(true);
    type.setStoreTermVectors(true);
    type.freeze();
    Token syn = new Token("httpwwwfacebookcom", 6, 29);
    syn.setPositionIncrement(0);
    CannedTokenStream ts = new CannedTokenStream(
        new Token("test", 0, 4),
        new Token("http", 6, 10),
        syn,
        new Token("www", 13, 16),
        new Token("facebook", 17, 25),

View Full Code Here

Examples of org.apache.lucene.analysis.CannedTokenStream

    doc.add( new Field( "field_super_exact", fieldValue, matched ) );  // Whitespace tokenized without toLower
    doc.add( new Field( "field_characters", fieldValue, matched ) );   // Each letter is a token
    doc.add( new Field( "field_tripples", fieldValue, matched ) );     // Every three letters is a token
    doc.add( new Field( "field_sliced", fieldValue.substring( 0,       // Sliced at 10 chars then analyzed just like field
      Math.min( fieldValue.length() - 1 , 10 ) ), matched ) );
    doc.add( new Field( "field_der_red", new CannedTokenStream(        // Hacky field containing "der" and "red" at pos = 0
          token( "der", 1, 0, 3 ),
          token( "red", 0, 0, 3 )
        ), matched ) );


    final Map<String, Analyzer> fieldAnalyzers = new TreeMap<String, Analyzer>();

View Full Code Here

Examples of org.apache.lucene.analysis.CannedTokenStream

        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
        
        return new TokenStreamComponents(tokenizer) {
          int tokenStreamCounter = 0;
          final TokenStream[] tokenStreams = new TokenStream[] {
            new CannedTokenStream(new Token[] {
                token("wifi",1,1),
                token("hotspot",0,2),
                token("network",1,1),
                token("is",1,1),
                token("slow",1,1)
              }),
            new CannedTokenStream(new Token[] {
                token("wi",1,1),
                token("hotspot",0,3),
                token("fi",1,1),
                token("network",1,1),
                token("is",1,1),
                token("fast",1,1)


              }),
            new CannedTokenStream(new Token[] {
                token("wifi",1,1),
                token("hotspot",0,2),
                token("network",1,1)
              }),
          };

View Full Code Here

Examples of org.apache.lucene.analysis.CannedTokenStream

        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
        
        return new TokenStreamComponents(tokenizer) {
          int tokenStreamCounter = 0;
          final TokenStream[] tokenStreams = new TokenStream[] {
            new CannedTokenStream(new Token[] {
                token("ab",1,1),
                token("ba",0,1),
                token("xc",1,1)
              }),
            new CannedTokenStream(new Token[] {
                token("ba",1,1),          
                token("xd",1,1)
              }),
            new CannedTokenStream(new Token[] {
                token("ab",1,1),
                token("ba",0,1),
                token("x",1,1)
              })
          };

View Full Code Here

Examples of org.apache.lucene.analysis.CannedTokenStream

          @Override
          public TokenStream getTokenStream() {
            // 4th time we are called, return tokens a b,
            // else just a:
            if (count++ != 3) {
              return new CannedTokenStream(new Token[] {
                  token("a", 1, 1),
                });
            } else {
              // After that "a b":
              return new CannedTokenStream(new Token[] {
                  token("a", 1, 1),
                  token("b", 1, 1),
                });
            }
          }

View Full Code Here

Examples of org.apache.lucene.analysis.CannedTokenStream

              int tokenStreamCounter = 0;
              final TokenStream[] tokenStreams = new TokenStream[] {
                new CannedBinaryTokenStream(new BinaryToken[] {
                    token(new BytesRef(new byte[] {0x61, (byte) 0xff, 0x61})),
                  }),
                new CannedTokenStream(new Token[] {
                    token("a",1,1),          
                    token("a",1,1)
                  }),
                new CannedTokenStream(new Token[] {
                    token("a",1,1),
                    token("a",1,1)
                  }),
                new CannedBinaryTokenStream(new BinaryToken[] {
                    token(new BytesRef(new byte[] {0x61, (byte) 0xff, 0x61})),

View Full Code Here

Examples of org.apache.lucene.analysis.CannedTokenStream

        
        return new TokenStreamComponents(tokenizer) {


          @Override
          public TokenStream getTokenStream() {
            return new CannedTokenStream(new Token[] {
                token("hairy", 1, 1),
                token("smelly", 0, 1),
                token("dog", 1, 1),
              });
          }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.