Package com.liveramp.cascading_ext

Examples of com.liveramp.cascading_ext.FixedSizeBitSet

110
111
112
113
114
115
116
117
118
119
    long splitSize = BloomUtil.getSplitSize(numBits, numSplits);

    Context c = new Context();

    for(int i = minHashes; i <= maxHashes; i++){
      c.numHashesToBitSet.put(i, new FixedSizeBitSet(splitSize, new byte[FixedSizeBitSet.getNumBytesToStore(splitSize)]));
    }

    call.setContext(c);
  }
View Full Code Here
40
41
42
43
44
45
46
  public BloomFilter() {
  }

  public BloomFilter(long vectorSize, int numHashes) {
    this(vectorSize, numHashes, new FixedSizeBitSet(vectorSize), 0);
  }
View Full Code Here
144
145
146
147
148
149
150
          " but current hash function type is " + hashFunction.getHashID() + "!");
    }

    byte[] bytes = new byte[FixedSizeBitSet.getNumBytesToStore(vectorSize)];
    in.readFully(bytes);
    bits = new FixedSizeBitSet(vectorSize, bytes);
  }
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
    // http://pages.cs.wisc.edu/~cao/papers/summary-cache/node8.html
    return Math.pow(1.0 - Math.exp((double) -numHashes * numElements / vectorSize), numHashes);
  }

  private static BloomFilter mergeBloomParts(String tapPath, long numBloomBits, long splitSize, int numBloomHashes, long numElems) throws IOException {
    FixedSizeBitSet bitSet = new FixedSizeBitSet(numBloomBits);

    if (FileSystemHelper.getFS().exists(new Path(tapPath))) {
      Hfs tap = new Hfs(new SequenceFile(new Fields("split", "filter")), tapPath);
      TupleEntryIterator itr = tap.openForRead(CascadingUtil.get().getFlowProcess());
      while (itr.hasNext()) {
        TupleEntry cur = itr.next();
        long split = cur.getLong(0);
        FixedSizeBitSet curSet = new FixedSizeBitSet(splitSize, ((BytesWritable) cur.getObject(1)).getBytes());
        for (long i = 0; i < curSet.numBits(); i++) {
          if (curSet.get(i)) {
            bitSet.set(split * splitSize + i);
          }
        }
      }
      itr.close();
View Full Code Here
TOP

Related Classes of com.liveramp.cascading_ext.FixedSizeBitSet

  • com.liveramp.cascading_ext.bloom.BloomFilter
  • com.liveramp.cascading_ext.bloom.BloomUtil
  • com.liveramp.cascading_ext.bloom.operation.CreateBloomFilterFromIndices
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.