Package ru.decipher.extraction.http.impl

Examples of ru.decipher.extraction.http.impl.BasicHttpRequest

211
212
213
214
215
216
217
218
219
220
221
222
    public void testThatRetrialSubmissionWillBePerformedWellInCaseOfActiveTaskLimitIsSet() throws Exception {
        processor = new GeneralProcessor<>(5);
        MockHttpHandlerWith3Retry handler = new MockHttpHandlerWith3Retry(processor, factory);

        for (int i = 0; i < 1000; i++) {
            BasicHttpRequest request = new BasicHttpRequest();
            request.setUrl("test://index");
            processor.submit(handler.buildTask(request));
        }
        processor.shutdown();
        assertEquals(1000, handler.getDone());
    }
View Full Code Here
225
226
227
228
229
230
231
232
233
234
235
    public void testThatRequestPriorityWorksWell() throws Exception {
        for (int i = 0; i < 1000; i++) {
            final MockHttpTask task = new MockHttpTask();
            task.setFactory(factory);
            task.registerHandler(handler);
            final BasicHttpRequest request = new BasicHttpRequest("test://index-" + i, 1);
            task.setRequest(request);
            processor.submit(task);
        }
        processor.shutdown();
        assertEquals(1000, handler.getHandledRequestsCount());
View Full Code Here
246
247
248
249
250
251
252
253
254
255
256
            }

            @Override
            public void run() {
                try {
                    processor.submit(handler.buildTask(new BasicHttpRequest("test://index?" + page)));
                    sleep(10);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
View Full Code Here
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
        processor = new GeneralProcessor<>(1);
        final MockHttpHandlerWithDelay delayedHandler = new MockHttpHandlerWithDelay(1000);
        MockHttpTask task = new MockHttpTask();
        task.setFactory(factory);
        task.registerHandler(delayedHandler);
        BasicHttpRequest request = new BasicHttpRequest("test://index", 1);
        task.setRequest(request);
        processor.submit(task);

        task = new MockHttpTask();
        task.setFactory(factory);
        task.registerHandler(delayedHandler);
        request = new BasicHttpRequest("test://index", 1);
        task.setRequest(request);
        processor.submit(task);
        long actual = processor.getTaskCount();
        processor.shutdown();
        assertEquals(2, actual);
View Full Code Here
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
        processor = new GeneralProcessor<>(1, 0, "duplicates test processor");
        final MockHttpHandlerWithDelay delayedHandler = new MockHttpHandlerWithDelay(1000);
        MockHttpTask task = new MockHttpTask();
        task.setFactory(factory);
        task.registerHandler(delayedHandler);
        BasicHttpRequest request = new BasicHttpRequest("test://index", 1);
        task.setRequest(request);
        processor.submit(task);

        sleep(2000);

        task = new MockHttpTask();
        task.setFactory(factory);
        task.registerHandler(delayedHandler);
        request = new BasicHttpRequest("test://index", 1);
        task.setRequest(request);
        processor.submit(task);
        long actual = processor.getTaskCount();
        processor.shutdown();
        assertEquals(1, actual);
View Full Code Here
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
    @Test(expected = ProcessorTaskSubmissionException.class)
    public void testThatDuplicatesAreNotAllowedToSubmitWhenZeroDuplicateLimitIsSet() throws Exception {
        processor = new GeneralProcessor<>(1, 0, "duplicates test processor");
        final MockHttpHandlerWithDelay delayedHandler = new MockHttpHandlerWithDelay(1000);
        MockHttpTask task;
        BasicHttpRequest request;

        task = new MockHttpTask();
        task.setFactory(factory);
        task.registerHandler(delayedHandler);
        request = new BasicHttpRequest("test://index", 1);
        task.setRequest(request);
        processor.submit(task);

        task = new MockHttpTask();
        task.setFactory(factory);
        task.registerHandler(delayedHandler);
        request = new BasicHttpRequest("test://index", 1);
        task.setRequest(request);
        processor.submit(task);
        processor.shutdown();
    }
View Full Code Here
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
    @Test(expected = ProcessorTaskSubmissionException.class)
    public void testThatDuplicatesNoExceedSpecifiedLimit() throws Exception {
        processor = new GeneralProcessor<>(1, 1, "duplicates test processor");
        final MockHttpHandlerWithDelay delayedHandler = new MockHttpHandlerWithDelay(1000);
        MockHttpTask task;
        BasicHttpRequest request;

        task = new MockHttpTask();
        task.setFactory(factory);
        task.registerHandler(delayedHandler);
        request = new BasicHttpRequest("test://index", 1);
        task.setRequest(request);
        processor.submit(task);

        task = new MockHttpTask();
        task.setFactory(factory);
        task.registerHandler(delayedHandler);
        request = new BasicHttpRequest("test://index", 1);
        task.setRequest(request);
        processor.submit(task);

        task = new MockHttpTask();
        task.setFactory(factory);
        task.registerHandler(delayedHandler);
        request = new BasicHttpRequest("test://index", 1);
        task.setRequest(request);
        processor.submit(task);
        processor.shutdown();
    }
View Full Code Here
246
247
248
249
250
251
252
253
254
255
256
            }

            @Override
            public void run() {
                try {
                    processor.submit(handler.buildTask(new BasicHttpRequest("test://index?" + page)));
                    sleep(10);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
View Full Code Here
TOP

Related Classes of ru.decipher.extraction.http.impl.BasicHttpRequest

  • ru.decipher.GeneralProcessorTest
  • ru.decipher.proxy.impl.RandomUserAgentManagerTest
  • ru.decipher.proxy.source.impl.GoogleCom
  • ru.decipher.proxy.source.ProxySourceSite
  • ru.decipher.Task
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.