Package org.apache.hadoop.util.bloom

Examples of org.apache.hadoop.util.bloom.Filter

44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
  public void aggregate(String data, String metadata) throws IOException, InterruptedException {
    // instantiate a bloom filter input key initialized by the data
    Key key = new Key(data.getBytes());

    // if the key is already in the filter, forget about it
    Filter filter = this.getFilter();
    if (filter.membershipTest(key))
      return;

    // add the key to the bloom filter
    filter.add(key);

    if (this.isCombining())
      this.collect(data);
    else
      this.total++;
View Full Code Here
30
31
32
33
34
35
36
37
38
39
40
41
              "_FUNC_(string bloom1, string bloom2) "
    )
public class BloomAndUDF extends UDF {

  public String evaluate( String bloom1Str, String bloom2Str ) throws IOException {
    Filter bloom1 = BloomFactory.GetBloomFilter( bloom1Str);
    Filter bloom2 = BloomFactory.GetBloomFilter( bloom2Str);
   
    bloom1.and( bloom2);
   
    return BloomFactory.WriteBloomToString( bloom1);
  }
View Full Code Here
31
32
33
34
35
36
37
38
39
40
41
42
              "_FUNC_(string bloom1, string bloom2) "
    )
public class BloomOrUDF extends UDF {

  public String evaluate( String bloom1Str, String bloom2Str ) throws IOException {
    Filter bloom1 = BloomFactory.GetBloomFilter( bloom1Str);
    Filter bloom2 = BloomFactory.GetBloomFilter( bloom2Str);
   
    bloom1.or( bloom2);
   
    return BloomFactory.WriteBloomToString( bloom1);
  }
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
   * @param returnEncoded
   * @return
   */
  public String evaluate( String mapFilename, Boolean returnEncoded) throws HiveException {
    try {
      Filter bloom = BloomFactory.GetNamedBloomFilter(mapFilename);
      if(bloom == null) {
        bloom = this.loadBloom(mapFilename);
        BloomFactory.PutNamedBloomFilter( mapFilename, bloom );
      }
      if( returnEncoded)   {
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
   *    parsed from UUencoded format.
   * @param name
   * @return
   */
  public static Filter GetBloomFilter( String str) {
    Filter bloom = GetNamedBloomFilter( str);
    if( bloom == null) {
      try {
        bloom = ReadBloomFromString( str);
        return bloom;
      } catch (IOException e) {
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
      stream.flush();
  }
 
  public static Filter ReadBloomFromString( String str) throws IOException {
    if( str != null ) {
      Filter filter = NewVesselBloom();
      byte[] decoded = Base64.decodeBase64( str.getBytes());
      DataInputStream dataInput = new DataInputStream( new ByteArrayInputStream(decoded));
   
      filter.readFields(dataInput);
      return filter;
    } else {
      return NewBloomInstance();
    }
  }
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
50
    )
public class BloomContainsUDF extends UDF {


  public Boolean evaluate( String key, String bloomFilter) throws HiveException {
    Filter bloom = BloomFactory.GetBloomFilter(bloomFilter);
    if( bloom != null) {
      return bloom.membershipTest( new Key(key.getBytes()));
    } else {
      throw new HiveException("Unable to find bloom " + bloomFilter);
    }
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.util.bloom.Filter

  • brickhouse.udf.bloom.BloomAndUDF
  • brickhouse.udf.bloom.BloomContainsUDF
  • brickhouse.udf.bloom.BloomFactory
  • brickhouse.udf.bloom.BloomNotUDF
  • brickhouse.udf.bloom.BloomOrUDF
  • brickhouse.udf.bloom.BloomTest
  • brickhouse.udf.bloom.BloomUDAF$BloomUDAFEvaluator
  • brickhouse.udf.bloom.DistributedBloomUDF
  • org.apache.mahout.utils.nlp.collocations.llr.BloomTokenFilterTest
  • sizzle.aggregators.UniqueAggregator
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.