Examples of org.apache.lucene.util.encoding.VInt8IntEncoder

org.apache.lucene.util.encoding.VInt8IntEncoder
An {@link IntEncoder} which implements variable length encoding. A number isencoded as follows:
- If it is less than 127 and non-negative, i.e. uses only 7 bits, it is encoded as a single byte: 0bbbbbbb.
- If it occupies more than 7 bits, it is represented as a series of bytes, each byte carrying 7 bits. All but the last byte have the MSB set, the last one has it unset.
Example:
1. n = 117 = 01110101: This has less than 8 significant bits, therefore is encoded as 01110101 = 0x75.
2. n = 100000 = (binary) 11000011010100000. This has 17 significant bits, thus needs three Vint8 bytes. Pad it to a multiple of 7 bits, then split it into chunks of 7 and add an MSB, 0 for the last byte, 1 for the others: 1|0000110 1|0001101 0|0100000 = 0x86 0x8D 0x20.
NOTE: although this encoder is not limited to values ≥ 0, it is not recommended for use with negative values, as their encoding will result in 5 bytes written to the output stream, rather than 4. For such values, either use {@link SimpleIntEncoder} or write your own version of variable lengthencoding, which can better handle negative values. @lucene.experimental

   * In any event when changing this value make sure you know what you are
   * doing, and test the results - e.g. counts, if the application is about
   * counting facets.
   */
  public IntEncoder createEncoder() {
    return new SortingIntEncoder(new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder())));
  }

View Full Code Here


  @Test
  public void testPayloadIntDecodingIterator() throws Exception {
    Directory dir = newDirectory();
    DataTokenStream dts = new DataTokenStream("1",new SortingIntEncoder(
        new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder()))));
    RandomIndexWriter writer = new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, 
        new MockAnalyzer(random, MockTokenizer.KEYWORD, false)).setMergePolicy(newLogMergePolicy()));
    for (int i = 0; i < data.length; i++) {
      dts.setIdx(i);
      Document doc = new Document();

View Full Code Here

   */
  @Test
  public void testPayloadIteratorWithInvalidDoc() throws Exception {
    Directory dir = newDirectory();
    DataTokenStream dts = new DataTokenStream("1",new SortingIntEncoder(
        new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder()))));
    DataTokenStream dts2 = new DataTokenStream("2",new SortingIntEncoder(
        new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder()))));
    // this test requires that no payloads ever be randomly present!
    final Analyzer noPayloadsAnalyzer = new Analyzer() {
      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new MockTokenizer(reader, MockTokenizer.KEYWORD, false);

View Full Code Here


    char[] separator = header.toCharArray();
    Arrays.fill(separator, '-');
    System.out.println(separator);


    encoderTest(new VInt8IntEncoder(), facetIDs, loopFactor);
    encoderTest(new SortingIntEncoder(new UniqueValuesIntEncoder(new VInt8IntEncoder())), facetIDs, loopFactor);
    encoderTest(new SortingIntEncoder(new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder()))), facetIDs, loopFactor);
    encoderTest(new SortingIntEncoder(new UniqueValuesIntEncoder(new DGapIntEncoder(new EightFlagsIntEncoder()))), facetIDs, loopFactor);
    encoderTest(new SortingIntEncoder(new UniqueValuesIntEncoder(new DGapIntEncoder(new FourFlagsIntEncoder()))), facetIDs, loopFactor);
    encoderTest(new SortingIntEncoder(new UniqueValuesIntEncoder(new DGapIntEncoder(new NOnesIntEncoder(3)))), facetIDs, loopFactor);
    encoderTest(new SortingIntEncoder(new UniqueValuesIntEncoder(new DGapIntEncoder(new NOnesIntEncoder(4)))), facetIDs, loopFactor);

View Full Code Here

    setData();
  }


  @Test
  public void testVInt8() throws Exception {
    encoderTest(new VInt8IntEncoder());
    
    // cover negative numbers;
    IntEncoder enc = new VInt8IntEncoder();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    enc.reInit(baos);
    enc.encode(-1);
    
    IntDecoder dec = enc.createMatchingDecoder();
    dec.reInit(new ByteArrayInputStream(baos.toByteArray()));
    assertEquals(-1, dec.decode());
  }

View Full Code Here

    encoderTest(new SimpleIntEncoder());
  }
  
  @Test
  public void testSortingUniqueValues() {
    encoderTest(new SortingIntEncoder(new UniqueValuesIntEncoder(new VInt8IntEncoder())));
  }

View Full Code Here

    encoderTest(new SortingIntEncoder(new UniqueValuesIntEncoder(new VInt8IntEncoder())));
  }


  @Test
  public void testSortingUniqueDGap() {
    encoderTest(new SortingIntEncoder(new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder()))));
  }

View Full Code Here


  @Test
  public void testPayloadIntDecodingIterator() throws Exception {
    Directory dir = newDirectory();
    DataTokenStream dts = new DataTokenStream("1",new SortingIntEncoder(
        new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder()))));
    RandomIndexWriter writer = new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, 
        new MockAnalyzer(random, MockTokenizer.KEYWORD, false)).setMergePolicy(newLogMergePolicy()));
    for (int i = 0; i < data.length; i++) {
      dts.setIdx(i);
      Document doc = new Document();

View Full Code Here

   */
  @Test
  public void testPayloadIteratorWithInvalidDoc() throws Exception {
    Directory dir = newDirectory();
    DataTokenStream dts = new DataTokenStream("1",new SortingIntEncoder(
        new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder()))));
    // this test requires that no payloads ever be randomly present!
    final Analyzer noPayloadsAnalyzer = new Analyzer() {
      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new MockTokenizer(reader, MockTokenizer.KEYWORD, false);

View Full Code Here

TOP

Related Classes of org.apache.lucene.util.encoding.VInt8IntEncoder

org.apache.lucene.facet.index.params.CategoryListParams

org.apache.lucene.facet.search.CategoryListIteratorTest

org.apache.lucene.util.encoding.EncodingSpeed

org.apache.lucene.util.encoding.EncodingTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.