Package org.apache.hadoop.util.bloom

Examples of org.apache.hadoop.util.bloom.BloomFilter

169
170
171
172
173
174
175
176
177
    }
   
    @Override
    public void initialize() {
      super.initialize();
      bloomFilter = new BloomFilter(vectorSize, nbHash, Hash.MURMUR_HASH);
      ptype.initialize(getConfiguration());
      keyToBytesFn = getKeyToBytesMapFn(ptype, getConfiguration());
    }
View Full Code Here
227
228
229
230
231
232
233
234
235
236
        iterable = bloomData.read(getContext());
      } catch (IOException e) {
        throw new CrunchRuntimeException("Error reading right-side of map side join: ", e);
      }

      bloomFilter = new BloomFilter(vectorSize, nbHash, Hash.MURMUR_HASH);
      for (BloomFilter subFilter : iterable) {
        bloomFilter.or(subFilter);
      }
    }
View Full Code Here
93
94
95
96
97
98
99
100
101
102
        }
        return list;
    }

    private void init() throws IOException {
        filter = new BloomFilter();
        String dcFile = "./" + getFilenameFromPath(bloomFile) +
            "/part-r-00000";
        filter.readFields(new DataInputStream(new FileInputStream(dcFile)));
    }
View Full Code Here
105
106
107
108
109
110
111
112
     * For testing only, do not use directly.
     */
    public void setFilter(DataByteArray dba) throws IOException {
        DataInputStream dis = new DataInputStream(new
            ByteArrayInputStream(dba.get()));
        filter = new BloomFilter();
        filter.readFields(dis);
    }
View Full Code Here
136
137
138
139
140
141
142
143
144
145
            byte[] b;
            if (t.size() == 1) b = DataType.toBytes(t.get(0));
            else b = DataType.toBytes(t, DataType.TUPLE);

            Key k = new Key(b);
            filter = new BloomFilter(vSize, numHash, hType);
            filter.add(k);

            return TupleFactory.getInstance().newTuple(bloomOut());
        }
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
        hType = convertHashType(hashType);
    }


    protected DataByteArray bloomOr(Tuple input) throws IOException {
        filter = new BloomFilter(vSize, numHash, hType);

        try {
            DataBag values = (DataBag)input.get(0);
            for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
                Tuple t = it.next();
View Full Code Here
105
106
107
108
109
110
111
112
113
    }

    protected BloomFilter bloomIn(DataByteArray b) throws IOException {
        DataInputStream dis = new DataInputStream(new
            ByteArrayInputStream(b.get()));
        BloomFilter f = new BloomFilter();
        f.readFields(dis);
        return f;
    }
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
152
153
    }
    assertEquals("Analyzer produced too few terms", expected.length, pos);
  }

  private static Filter getFilter(String[] tokens) throws IOException {
    Filter filter = new BloomFilter(100,50, Hash.JENKINS_HASH);
    Key k = new Key();
    for (String s: tokens) {
      setKey(k,s);
      filter.add(k);
    }
    return filter;
  }
View Full Code Here
168
169
170
171
172
173
174
175
176
    }
   
    @Override
    public void initialize() {
      super.initialize();
      bloomFilter = new BloomFilter(vectorSize, nbHash, Hash.MURMUR_HASH);
      ptype.initialize(getConfiguration());
      keyToBytesFn = getKeyToBytesMapFn(ptype, getConfiguration());
    }
View Full Code Here
226
227
228
229
230
231
232
233
234
235
        iterable = bloomData.read(getContext());
      } catch (IOException e) {
        throw new CrunchRuntimeException("Error reading right-side of map side join: ", e);
      }

      bloomFilter = new BloomFilter(vectorSize, nbHash, Hash.MURMUR_HASH);
      for (BloomFilter subFilter : iterable) {
        bloomFilter.or(subFilter);
      }
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.util.bloom.BloomFilter

  • brickhouse.udf.bloom.BloomFactory
  • cc.bf.NgramsToBloomFilter$BFBase
  • cc.bf.ReadNgram
  • com.cloudera.util.bloom.BloomSet
  • com.manning.hip.ch7.bloom.BloomFilterDumper
  • com.ontology2.bakemono.bloom.BloomReducer
  • com.ontology2.bakemono.bloom.BloomReducerTest
  • mrdp.appendixA.BloomFilterDriver
  • org.apache.crunch.contrib.bloomfilter.BloomFilterFn
  • org.apache.crunch.contrib.bloomfilter.BloomFiltersIT
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.