Examples of org.sindice.siren.analysis.filter.DatatypeAnalyzerFilter

Package org.sindice.siren.analysis.filter

Examples of org.sindice.siren.analysis.filter.DatatypeAnalyzerFilter

org.sindice.siren.analysis.filter.DatatypeAnalyzerFilter
This class performs post-processing operation on the tokens extracted by {@link JsonTokenizer} based on the {@link DatatypeAttribute}.
This filter provides a {@link #register(char[],Analyzer)} method which allowsto register an {@link Analyzer} to a specific datatype.


  @Override
  protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
    final JsonTokenizer source = new JsonTokenizer(reader);


    final DatatypeAnalyzerFilter tt = new DatatypeAnalyzerFilter(matchVersion, source, fieldAnalyzer, valueAnalyzer);
    for (final Entry<Object, Analyzer> e : regAnalyzers.entrySet()) {
      tt.register((char[]) e.getKey(), e.getValue());
    }
    TokenStream sink = new PositionAttributeFilter(tt);
    sink = new SirenPayloadFilter(sink);
    return new TokenStreamComponents(source, sink);
  }

View Full Code Here

  protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
    final TupleTokenizer source = new TupleTokenizer(reader);


    TokenStream sink = new TokenTypeFilter(source, new int[] {TupleTokenizer.BNODE,
                                                              TupleTokenizer.DOT});
    final DatatypeAnalyzerFilter tt = new DatatypeAnalyzerFilter(matchVersion, sink, anyURIAnalyzer, stringAnalyzer);
    for (final Entry<Object, Analyzer> e : regLitAnalyzers.entrySet()) {
      tt.register((char[]) e.getKey(), e.getValue());
    }
    sink = new PositionAttributeFilter(tt);
    sink = new SirenPayloadFilter(sink);
    return new TokenStreamComponents(source, sink);
  }

View Full Code Here

      @Override
      protected TokenStreamComponents createComponents(final String fieldName,
                                                       final Reader reader) {
        final Version matchVersion = Version.LUCENE_40;
        final JsonTokenizer src = new JsonTokenizer(reader);
        TokenStream tok = new DatatypeAnalyzerFilter(matchVersion, src,
          new StandardAnalyzer(matchVersion),
          new StandardAnalyzer(matchVersion));
        tok = new LowerCaseFilter(matchVersion, tok);
        // The PositionAttributeFilter and SirenPayloadFilter are mandatory
        // and must be always the last filters in your token stream

View Full Code Here

    }
  }


  @Override
  public TokenStream create(final TokenStream input) {
    final DatatypeAnalyzerFilter f = new DatatypeAnalyzerFilter(luceneDefaultVersion, input);


    for (final String datatype : analyzers.keySet()) {
      f.register(datatype.toCharArray(), analyzers.get(datatype));
    }


    return f;
  }

View Full Code Here

TOP

Related Classes of org.sindice.siren.analysis.filter.DatatypeAnalyzerFilter

org.apache.lucene.analysis.Analyzer

org.apache.lucene.util.IntsRef

org.sindice.siren.analysis.JsonAnalyzer

org.sindice.siren.analysis.TupleAnalyzer

org.sindice.siren.demo.SimpleIndexer

org.sindice.siren.solr.analysis.DatatypeAnalyzerFilterFactory

org.sindice.siren.util.ReusableCharArrayReader

java.io.IOException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.