Package kafka.api

Examples of kafka.api.FetchRequest

47
48
49
50
51
52
53
54
55
56
57
    while(true)
    {
      List<FetchRequest> list = new ArrayList<FetchRequest>();
      for(int i=0 ; i < numParts; i++)
      {
        FetchRequest req = new FetchRequest(topic, i, offset, fetchSize);
        list.add(req);
      }
      MultiFetchResponse response = simpleConsumer.multifetch(list);
      if(response.hasNext())
      {
View Full Code Here
193
194
195
196
197
198
199
200
201
202
203
    while (toContinue()) {

      // fetch data using multi-fetch interfaces
      // TODO: change to high-level interface once it supports "reset"
      FetchRequest fetchRequest = new FetchRequest(_topic, _partition,
          _offset, _bufferSize);

      List<FetchRequest> array = new ArrayList<FetchRequest>();
      array.add(fetchRequest);
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
                                                       KafkaProperties.kafkaServerPort,
                                                       KafkaProperties.connectionTimeOut,
                                                       KafkaProperties.kafkaProducerBufferSize);

    System.out.println("Testing single fetch");
    FetchRequest req = new FetchRequest(KafkaProperties.topic2, 0, 0L, 100);
    ByteBufferMessageSet messageSet = simpleConsumer.fetch(req);
    printMessages(messageSet);

    System.out.println("Testing single multi-fetch");
    req = new FetchRequest(KafkaProperties.topic2, 0, 0L, 100);
    List<FetchRequest> list = new ArrayList<FetchRequest>();
    list.add(req);
    req = new FetchRequest(KafkaProperties.topic3, 0, 0L, 100);
    list.add(req);
    MultiFetchResponse response = simpleConsumer.multifetch(list);
    int fetchReq = 0;
    while(response.hasNext())
    {
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
                                                       KafkaProperties.connectionTimeOut,
                                                       KafkaProperties.kafkaProducerBufferSize,
                                                       KafkaProperties.clientId);

    System.out.println("Testing single fetch");
    FetchRequest req = new FetchRequestBuilder()
            .clientId(KafkaProperties.clientId)
            .addFetch(KafkaProperties.topic2, 0, 0L, 100)
            .build();
    FetchResponse fetchResponse = simpleConsumer.fetch(req);
      printMessages((ByteBufferMessageSet) fetchResponse.messageSet(KafkaProperties.topic2, 0));
View Full Code Here
148
149
150
151
152
153
154
155
156
157
158
    }
   
    public boolean fetchMore () throws IOException {
        if (!hasMore()) return false;

        FetchRequest fetchRequest = builder
                .clientId(_request.clientId())
                .addFetch(_request.getTopic(), _request.getPartition(), _offset, _bufferSize)
                .build();

        long tempTime = System.currentTimeMillis();
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
                                                       KafkaProperties.kafkaServerPort,
                                                       KafkaProperties.connectionTimeOut,
                                                       KafkaProperties.kafkaProducerBufferSize);

    System.out.println("Testing single fetch");
    FetchRequest req = new FetchRequest(KafkaProperties.topic2, 0, 0L, 100);
    ByteBufferMessageSet messageSet = simpleConsumer.fetch(req);
    printMessages(messageSet);

    System.out.println("Testing single multi-fetch");
    req = new FetchRequest(KafkaProperties.topic2, 0, 0L, 100);
    List<FetchRequest> list = new ArrayList<FetchRequest>();
    list.add(req);
    req = new FetchRequest(KafkaProperties.topic3, 0, 0L, 100);
    list.add(req);
    MultiFetchResponse response = simpleConsumer.multifetch(list);
    int fetchReq = 0;
    for (ByteBufferMessageSet resMessageSet : response )
    {
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
        if (value == null) {
            value = new BytesWritable();
        }

        if (messages == null) {
            FetchRequest request = new FetchRequest(topic, partition, watermark, fetchSize);
            log.info("{} fetching offset {} ", topic+":" + split.getBrokerId() +":" + partition, watermark);
            messages = consumer.fetch(request);
            if (messages.getErrorCode() == ErrorMapping.OffsetOutOfRangeCode())
            {
                log.info("Out of bounds = " + watermark);
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
    for (int i = 0; i < MAX_KAFKA_FETCH_RETRIES; ++i) {
      if (consumer == null) {
          findLeader();
      }

      FetchRequest req = new FetchRequestBuilder()
        .clientId(clientName)
        .addFetch(topic, partition, fetchOffset, BUFFER_SIZE_BYTES)
        .maxWait(fetchTimeoutMs)
        .build();
      FetchResponse fetchResponse = consumer.fetch(req);
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
            String instanceId = (String) meta.get("instanceId");
            if(!_config.forceFromStart || instanceId.equals(_topologyInstanceId)) {
                SimpleConsumer consumer = _connections.getConsumer(partition);
                long offset = (Long) meta.get("offset");
                long nextOffset = (Long) meta.get("nextOffset");
                ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(_config.topic, partition % partitionsPerHost, offset, _config.fetchSizeBytes));
                for(MessageAndOffset msg: msgs) {
                    if(offset == nextOffset) break;
                    if(offset > nextOffset) {
                        throw new RuntimeException("Error when re-emitting batch. overshot the end offset");
                    }
View Full Code Here
31
32
33
34
35
36
37
38
39
40
41
             if(config.forceFromStart) startTime = config.startOffsetTime;
             offset = consumer.getOffsetsBefore(config.topic, partition % hosts.partitionsPerHost, startTime, 1)[0];
         }
         ByteBufferMessageSet msgs;
         try {
            msgs = consumer.fetch(new FetchRequest(config.topic, partition % hosts.partitionsPerHost, offset, config.fetchSizeBytes));
         } catch(Exception e) {
             if(e instanceof ConnectException) {
                 throw new FailedFetchException(e);
             } else {
                 throw new RuntimeException(e);
View Full Code Here
TOP

Related Classes of kafka.api.FetchRequest

  • co.cask.cdap.logging.kafka.KafkaConsumer
  • co.gridport.kafka.hadoop.KafkaInputRecordReader
  • com.alibaba.jstorm.kafka.MessageConsumer
  • com.facebook.presto.kafka.KafkaRecordSet$KafkaRecordCursor
  • kafka.etl.KafkaETLContext
  • kafka.etl.KafkaETLMapper
  • kafka.examples.SimpleConsumerDemo
  • kafka.perf.consumer.SimplePerfConsumer
  • org.apache.twill.internal.kafka.client.SimpleKafkaConsumer$ConsumerThread
  • storm.kafka.KafkaUtils
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.