// check number of years
long numberOfYears = remainingPeriod / 360L / 24L / 60L / 60L / 1000L;
int maxYears = maxYearsOfHistoricalData();
if (numberOfYears > maxYears) {
throw new RequestFailedException("Current number of market data lines only allows for " + maxYears + " of historical data");
}
// break request in smaller batches to comply with IB limitations
long maxPeriodForOneRequest = remainingPeriod;
long barsPerInterval = maxPeriodForOneRequest / barSize.getDurationInMs();
if (barsPerInterval > maxBarPerRequest) {
maxPeriodForOneRequest = barSize.getDurationInMs() * maxBarPerRequest;
}
maxPeriodForOneRequest = Math.min(maxPeriodForOneRequest, maxRequestPeriodPerBarSize(barSize));
Date nextEndDateTime = endDateTime;
// comply with IB timing restrictions
long lastBatchTime = 0L;
int requestNo = 0;
int approxCallNo = (int) (remainingPeriod / maxPeriodForOneRequest);
if (taskMonitor != null) {
taskMonitor.beginTask(approxCallNo * 10); // retrieving then adding partial results: IB calls count 7 - saving results count 3
}
logger.log(Level.INFO, "Start downloading: " + new Date()); // TODO: remove
boolean forceSleep = false;
while (remainingPeriod > 0) {
if (taskMonitor != null && taskMonitor.isCancelled()) {
throw new RequestFailedException("Request cancelled by user");
}
if (requestNo % numOfReqsInHistDataReqBatches == 0 || forceSleep) {
long elapsedSinceLastBatch = System.currentTimeMillis() - lastBatchTime;
logger.log(Level.INFO, "Now: " + timestampDateFormat.format(new Date()) + "; Elapsed: " + elapsedSinceLastBatch); // TODO: remove
if (elapsedSinceLastBatch < minElapsedBetweenHistDataReqBatches) {
try {
logger.log(Level.INFO, "Sleeping: " + (minElapsedBetweenHistDataReqBatches - elapsedSinceLastBatch) + "ms"); // TODO: remove
for (long partElapsed = 0L; partElapsed < minElapsedBetweenHistDataReqBatches - elapsedSinceLastBatch; partElapsed += HIST_DATA_REQUEST_SLEEP_QUANTUM) {
Thread.sleep(HIST_DATA_REQUEST_SLEEP_QUANTUM);
if (taskMonitor != null && taskMonitor.isCancelled()) {
throw new RequestFailedException("Request cancelled by user");
}
}
} catch (InterruptedException e) {
throw new RequestFailedException("Exception while sleeping through request batches", e);
}
}
lastBatchTime = System.currentTimeMillis();
forceSleep = false;
}
long currentRequestPeriod;
if (remainingPeriod > maxPeriodForOneRequest) {
currentRequestPeriod = maxPeriodForOneRequest;
} else if (remainingPeriod < minHistDataPeriodMs) {
currentRequestPeriod = minHistDataPeriodMs;
} else {
currentRequestPeriod = remainingPeriod;
}
TimePeriod requestPeriod = TimePeriod.findApproxPeriod(currentRequestPeriod);
List<IOHLCPoint> partialResult = null;
logger.log(Level.INFO, "Request #" + requestNo + ". Data until: " + nextEndDateTime); // TODO: remove
try {
partialResult = doGetHistoricalData(contract, nextEndDateTime, requestPeriod, barSize, dataType, includeAfterHours);
} catch (PacingViolationException e) {
logger.log(Level.WARNING, "Pacing violation exception while retrieving historical data from IB. Force sleep", e);
forceSleep = true;
continue;
} catch (NoDataReturnedException e) {
logger.log(Level.INFO, "Historical data request from IB returned no data. Continue with next request", e);
partialResult = null;
}
if (partialResult == null || nextEndDateTime.equals(partialResult.get(0).getIndex())) { // empty or duplicate result
logger.log(Level.INFO, "Empty or duplicate result: " + nextEndDateTime); // TODO: remove
nextEndDateTime = TimePeriod.subtractPeriodFromDate(nextEndDateTime, requestPeriod);
} else {
tmpResults.addFirst(partialResult);
nextEndDateTime = partialResult.get(0).getIndex();
}
remainingPeriod = nextEndDateTime.getTime() - startDateTime.getTime();
requestNo++;
if (taskMonitor != null) {
taskMonitor.worked(7);
}
}
logger.log(Level.INFO, "All data downloaded. Create time series: " + new Date()); // TODO: remove
List<IOHLCPoint> result = new ArrayList<IOHLCPoint>();
IOHLCPoint lastBarAdded = null;
Date dateOfLastBarAdded = new Date(0L);
for (List<IOHLCPoint> partialResult : tmpResults) {
if (taskMonitor != null && taskMonitor.isCancelled()) {
throw new RequestFailedException("Request cancelled by user");
}
for (IOHLCPoint bar : partialResult) {
Date endOfLastBarAdded = new Date(dateOfLastBarAdded.getTime() + barSize.getDurationInMs());
if (!bar.getIndex().before(startDateTime)) {