Package com.linkedin.databus2.core.container.request

Examples of com.linkedin.databus2.core.container.request.RegisterResponseEntry


  throws IOException, RequestProcessingException
  {
    Set<String> sources = getSourcesParam(request);
    if(sources == null || sources.size() != 1)
    {
      throw new RequestProcessingException("start requires exactly one source be specified");
    }
    long scn = request.getOptionalLongParam(PARAM_SCN, -1L);

    for(EventProducer producer : _eventProducers)
    {
View Full Code Here


        if (null != bootstrapStatsCollector)
        {
          bootstrapStatsCollector.registerErrBootstrap();
        }

        throw new RequestProcessingException(e);
      }

      DatabusComponentStatus componentStatus = _componentStatus.getStatusSnapshot();
      if (!componentStatus.isRunningStatus())
      {
        if (null != bootstrapStatsCollector)
          bootstrapStatsCollector.registerErrBootstrap();

       throw new RequestProcessingException(componentStatus.getMessage());
      }

      String partitionInfoString = request.getParams().getProperty(PARTITION_INFO_PARAM);

      DbusKeyFilter keyFilter = null;
      if ( (null != partitionInfoString) && (!partitionInfoString.isEmpty()))
      {
        try
        {
          keyFilter = KeyFilterConfigJSONFactory.parseDbusKeyFilter(partitionInfoString);
          if ( isDebug) LOG.debug("ServerSideFilter is :" + keyFilter);
        } catch ( Exception ex) {
          String msg = "Unable to parse partitionInfo from request. PartitionInfo was :" + partitionInfoString;
          LOG.error(msg,ex);
          throw new RequestProcessingException(msg,ex);
        }
      }

      String outputFormat = request.getParams().getProperty(OUTPUT_PARAM);
      Encoding enc = Encoding.BINARY;

      if ( null != outputFormat)
      {
        try
        {
          enc = Encoding.valueOf(outputFormat.toUpperCase());
        } catch (Exception ex) {
          LOG.error("Unable to find the output format for bootstrap request for " + outputFormat + ". Using Binary!!", ex);
        }
      }

      processor.setKeyFilter(keyFilter);
      String checkpointString = request.getRequiredStringParam(CHECKPOINT_PARAM);

      int bufferMarginSpace = DEFAULT_BUFFER_MARGIN_SPACE;
      if ( null != _serverHostPort)
      {
        bufferMarginSpace = Math.max(bufferMarginSpace, (_serverHostPort.length() + Checkpoint.BOOTSTRAP_SERVER_INFO.length() + DEFAULT_JSON_OVERHEAD_BYTES));
      }

      int clientFreeBufferSize = request.getRequiredIntParam(BATCHSIZE_PARAM) - checkpointString.length() - bufferMarginSpace;

        BootstrapEventWriter writer = null;
        if(_config.getPredicatePushDown())
          writer = createEventWriter(request, clientFreeBufferSize, null, enc);
        else
          writer = createEventWriter(request, clientFreeBufferSize, keyFilter, enc);

        Checkpoint cp = new Checkpoint(checkpointString);


      DbusClientMode consumptionMode = cp.getConsumptionMode();

      LOG.info("Bootstrap request received: " +
          "fetchSize=" + clientFreeBufferSize +
          ", consumptionMode=" + consumptionMode +
          ", checkpoint=" + checkpointString +
          ", predicatePushDown=" + _config.getPredicatePushDown()
          );

      try
      {
        boolean phaseCompleted = false;
        switch (consumptionMode)
        {
        case BOOTSTRAP_SNAPSHOT:
          phaseCompleted = processor.streamSnapShotRows(new Checkpoint(
              checkpointString), writer);
          break;
        case BOOTSTRAP_CATCHUP:
          phaseCompleted = processor.streamCatchupRows(new Checkpoint(
              checkpointString), writer);
          break;
        default:
          if (null != bootstrapStatsCollector)
            bootstrapStatsCollector.registerErrBootstrap();

          throw new RequestProcessingException("Unexpected mode: "
              + consumptionMode);
        }

        if (null != bootstrapStatsCollector)
          bootstrapStatsCollector.registerBootStrapReq(cp, System.currentTimeMillis()-startTime, clientFreeBufferSize);

        if (writer.getNumRowsWritten() == 0 && writer.getSizeOfPendingEvent() > 0)
        {
          // Append a header to indicate to the client that we do have at least one event to
          // send, but it is too large to fit into client's offered buffer.
          request.getResponseContent().addMetadata(DatabusHttpHeaders.DATABUS_PENDING_EVENT_SIZE,
                                                   writer.getSizeOfPendingEvent());
          if (isDebug)
          {
            LOG.debug("Returning 0 events but have pending event of size " + writer.getSizeOfPendingEvent());
          }
        }
        if (phaseCompleted)
        {
          request.getResponseContent().setMetadata(BootstrapProcessor.PHASE_COMPLETED_HEADER_NAME, BootstrapProcessor.PHASE_COMPLETED_HEADER_TRUE);
        }

      }
      catch (BootstrapDatabaseTooOldException e)
      {
        if (null != bootstrapStatsCollector)
          bootstrapStatsCollector.registerErrDatabaseTooOld();

        LOG.error("Bootstrap database is too old!", e);
        throw new RequestProcessingException(e);
      }
      catch (BootstrapDBException e)
      {
        if (null != bootstrapStatsCollector)
          bootstrapStatsCollector.registerErrBootstrap();
        throw new RequestProcessingException(e);
      }
      catch (SQLException e)
      {
        if (null != bootstrapStatsCollector)
          bootstrapStatsCollector.registerErrSqlException();

        throw new RequestProcessingException(e);
      }
      catch (BootstrapProcessingException e)
      {
        if (null != bootstrapStatsCollector)
          bootstrapStatsCollector.registerErrBootstrap();

        throw new RequestProcessingException(e);
      }
    }
    finally
    {
      if ( null != processor)
View Full Code Here

  protected void initializeBootstrapServerCommandProcessors() throws DatabusException
  {
    LOG.info("Initializing Bootstrap HTTP Server");
    LOG.info("Config=" + _bootstrapServerConfig);
    try{
      RequestProcessorRegistry processorRegistry = getProcessorRegistry();
      processorRegistry.register(ConfigRequestProcessor.COMMAND_NAME,
                                 new ConfigRequestProcessor(null, this));
      processorRegistry.register(BootstrapRequestProcessor.COMMAND_NAME,
                                 new BootstrapRequestProcessor(null, _bootstrapServerConfig, this));
      processorRegistry.register(StartSCNRequestProcessor.COMMAND_NAME,
                                 new StartSCNRequestProcessor(null, _bootstrapServerConfig, this));
      processorRegistry.register(TargetSCNRequestProcessor.COMMAND_NAME,
                                 new TargetSCNRequestProcessor(null, _bootstrapServerConfig, this));
      processorRegistry.register(ContainerOperationProcessor.COMMAND_NAME,
                                 new ContainerOperationProcessor(null, this));
    }
    catch (SQLException sqle)
    {
      throw new DatabusException("command registration failed", sqle);
View Full Code Here

      _csEventRequestProcessor = new ControlSourceEventsRequestProcessor(
          null, this, plist);
    else
      _csEventRequestProcessor.addEventProducers(plist);

    RequestProcessorRegistry processorRegistry = getProcessorRegistry();
    processorRegistry.reregister(
        ControlSourceEventsRequestProcessor.COMMAND_NAME,
        _csEventRequestProcessor);
  }
View Full Code Here

       throw new RequestProcessingException(componentStatus.getMessage());
      }

      String partitionInfoString = request.getParams().getProperty(PARTITION_INFO_PARAM);

      DbusKeyFilter keyFilter = null;
      if ( (null != partitionInfoString) && (!partitionInfoString.isEmpty()))
      {
        try
        {
          keyFilter = KeyFilterConfigJSONFactory.parseDbusKeyFilter(partitionInfoString);
View Full Code Here

            .andReturn(vs)
            .anyTimes();
    EasyMock.expect(srs.fetchLatestVersionedSchemaBySourceName(null)).andReturn(vs);

    // mock for MaxSCNReadWriter
    MaxSCNReaderWriter mscn = EasyMock.createMock(MaxSCNReaderWriter.class);
    EasyMock.expect(mscn.getMaxScn()).andReturn((long) -2).atLeastOnce();
    mscn.saveMaxScn(EasyMock.anyLong());
    EasyMock.expectLastCall().anyTimes();
    EasyMock.replay(mscn);
    EasyMock.replay(srs);

    int totalTransWritten = 0;
View Full Code Here

    // Register a command to allow start/stop/status of the relay
    List<EventProducer> plist = new ArrayList<EventProducer>();

    PhysicalPartition pPartition = pConfig.getPhysicalPartition();
    MaxSCNReaderWriter maxScnReaderWriters = _maxScnReaderWriters
        .getOrCreateHandler(pPartition);
    LOG.info("Starting server container with maxScnReaderWriter:"
        + maxScnReaderWriters);

    // Get the event buffer
View Full Code Here

  /** get maxScnReaderWriters given a physical source **/
  public MaxSCNReaderWriter getMaxSCNReaderWriter(PhysicalSourceStaticConfig pConfig)
  {
    try
    {
      MaxSCNReaderWriter maxScnReaderWriters = _maxScnReaderWriters
        .getOrCreateHandler(pConfig.getPhysicalPartition());
      return maxScnReaderWriters;
    }
    catch (DatabusException e)
    {
View Full Code Here

      PhysicalSourceStaticConfig[] pConfigs) throws IOException,
      InvalidConfigException, DatabusException {
    super(config, pConfigs);
    SequenceNumberHandlerFactory handlerFactory = _relayStaticConfig
        .getDataSources().getSequenceNumbersHandler().createFactory();
    _maxScnReaderWriters = new MultiServerSequenceNumberHandler(
        handlerFactory);
    _producers = new HashMap<PhysicalPartition, EventProducer>(
        _pConfigs.size());
    _monitoringProducers = new HashMap<PhysicalPartition, MonitoringEventProducer>(_pConfigs.size());
    _dbPullerStart = false;
View Full Code Here

      Assert.assertEquals("Number of Source batches", 2, result.size());
      Assert.assertEquals("Result Set Size", result.get(0).getDbUpdatesSet().size(), 1);
      Assert.assertEquals("Result Set Size", result.get(1).getDbUpdatesSet().size(), 1);

      DBUpdateImage dbi = result.get(0).getDbUpdatesSet().iterator().next();
      DBUpdateImage expDbi = newDbUpdates.get(0).getDbUpdatesSet().iterator().next();
      Assert.assertEquals("SCN Check", scn, dbi.getScn());
      String k = (String) dbi.getGenericRecord().get("name1");
      String v = (String) dbi.getGenericRecord().get("name2");
      String expK = (String) expDbi.getGenericRecord().get("name1");
      String expV = (String) expDbi.getGenericRecord().get("name2");
      Assert.assertEquals("Key Check", expK, k);
      Assert.assertEquals("Val Check", expV, v);

      dbi = result.get(1).getDbUpdatesSet().iterator().next();
      expDbi = newDbUpdates.get(1).getDbUpdatesSet().iterator().next();
      Assert.assertEquals("SCN Check", scn, dbi.getScn());
      k = (String) dbi.getGenericRecord().get("name1");
      v = (String) dbi.getGenericRecord().get("name2");
      expK = (String) expDbi.getGenericRecord().get("name1");
      expV = (String) expDbi.getGenericRecord().get("name2");
      Assert.assertEquals("Key Check", expK, k);
      Assert.assertEquals("Val Check", expV, v);
    }

    // Case when 2 txn have same source but different keys
    {
      String key = "key1";
      String oldVal = "val1";
      Map<String, String> keyVals1 = new HashMap<String, String>();
      keyVals1.put(key, oldVal);
      Map<String, String> keyVals2 = new HashMap<String, String>();
      key = "key2";
      String newVal = "val2";
      keyVals2.put(key, newVal);
      List<TransactionState.PerSourceTransactionalUpdate> oldDbUpdates =
          generateUpdatesForSchema2(sourceIds, keyVals1, scn);
      List<TransactionState.PerSourceTransactionalUpdate> newDbUpdates =
          generateUpdatesForSchema2(sourceIds, keyVals2, scn);
      List<TransactionState.PerSourceTransactionalUpdate> result =
          DBUpdatesMergeUtils.mergeTransactionData(newDbUpdates, oldDbUpdates);

      Assert.assertEquals("Number of Source batches", 2, result.size());
      Assert.assertEquals("Result Set Size", result.get(0).getDbUpdatesSet().size(), 2);
      Assert.assertEquals("Result Set Size", result.get(1).getDbUpdatesSet().size(), 2);

      Iterator<DBUpdateImage> itr = result.get(0).getDbUpdatesSet().iterator();
      DBUpdateImage dbi = itr.next();
      Assert.assertEquals("SCN Check", scn, dbi.getScn());
      String k = (String) dbi.getGenericRecord().get("name1");
      String v = (String) dbi.getGenericRecord().get("name2");
      dbi = itr.next();
      Assert.assertEquals("SCN Check", scn, dbi.getScn());
      String k2 = (String) dbi.getGenericRecord().get("name1");
      String v2 = (String) dbi.getGenericRecord().get("name2");

      if (keyVals1.containsKey(k))
      {
        Assert.assertEquals("Val Check", keyVals1.get(k), v);
        Assert.assertEquals("Val Check 2", keyVals2.get(k2), v2);
      }
      else
      {
        Assert.assertEquals("Val Check", keyVals2.get(k), v);
        Assert.assertEquals("Val Check 2", keyVals1.get(k2), v2);
      }

      itr = result.get(1).getDbUpdatesSet().iterator();
      dbi = itr.next();
      Assert.assertEquals("SCN Check", scn, dbi.getScn());
      k = (String) dbi.getGenericRecord().get("name1");
      v = (String) dbi.getGenericRecord().get("name2");
      dbi = itr.next();
      Assert.assertEquals("SCN Check", scn, dbi.getScn());
      k2 = (String) dbi.getGenericRecord().get("name1");
      v2 = (String) dbi.getGenericRecord().get("name2");

      if (keyVals1.containsKey(k))
      {
        Assert.assertEquals("Val Check", keyVals1.get(k), v);
        Assert.assertEquals("Val Check 2", keyVals2.get(k2), v2);
      }
      else
      {
        Assert.assertEquals("Val Check", keyVals2.get(k), v);
        Assert.assertEquals("Val Check 2", keyVals1.get(k2), v2);
      }
    }

    // Case when 2 txns have different sources
    {
      String key = "key1";
      String oldVal = "val1";
      Map<String, String> keyVals1 = new HashMap<String, String>();
      keyVals1.put(key, oldVal);
      Map<String, String> keyVals2 = new HashMap<String, String>();
      key = "key2";
      String newVal = "val2";
      keyVals2.put(key, newVal);
      List<TransactionState.PerSourceTransactionalUpdate> oldDbUpdates =
          generateUpdatesForSchema2(new short[] { sourceIds[0] }, keyVals1, scn);
      List<TransactionState.PerSourceTransactionalUpdate> newDbUpdates =
          generateUpdatesForSchema2(new short[] { sourceIds[1] }, keyVals2, scn);
      List<TransactionState.PerSourceTransactionalUpdate> result =
          DBUpdatesMergeUtils.mergeTransactionData(newDbUpdates, oldDbUpdates);

      Assert.assertEquals("Number of Source batches", 2, result.size());
      Assert.assertEquals("Result Set Size", result.get(0).getDbUpdatesSet().size(), 1);
      Assert.assertEquals("Result Set Size", result.get(1).getDbUpdatesSet().size(), 1);

      Iterator<DBUpdateImage> itr = result.get(0).getDbUpdatesSet().iterator();
      DBUpdateImage dbi = itr.next();
      Assert.assertEquals("SCN Check", scn, dbi.getScn());
      String k = (String) dbi.getGenericRecord().get("name1");
      String v = (String) dbi.getGenericRecord().get("name2");
      Assert.assertEquals("Val Check", keyVals1.get(k), v);

      itr = result.get(1).getDbUpdatesSet().iterator();
      dbi = itr.next();
      Assert.assertEquals("SCN Check", scn, dbi.getScn());
      k = (String) dbi.getGenericRecord().get("name1");
      v = (String) dbi.getGenericRecord().get("name2");
      Assert.assertEquals("Val Check", keyVals2.get(k), v);
    }
  }
View Full Code Here

TOP

Related Classes of com.linkedin.databus2.core.container.request.RegisterResponseEntry

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.