Package io.crate.blob.v2

Examples of io.crate.blob.v2.BlobIndexModule


            IndicesService indicesService = mock(IndicesService.class);
            bind(IndicesService.class).toInstance(indicesService);
            bind(Settings.class).toInstance(ImmutableSettings.EMPTY);

            BlobIndices blobIndices = new BlobIndices(
                    ImmutableSettings.EMPTY,
                    mock(TransportCreateIndexAction.class),
                    mock(TransportDeleteIndexAction.class),
                    mock(TransportUpdateSettingsAction.class),
                    indicesService,
View Full Code Here


        assertEquals(false, response.rows()[0][2]);
    }

    @Test
    public void testTableNameBlobTable() throws Exception {
        BlobIndices blobIndices = cluster().getInstance(BlobIndices.class);
        Settings indexSettings = ImmutableSettings.builder()
                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
                .build();
        blobIndices.createBlobTable("blobs", indexSettings).get();
        ensureGreen();

        execute("select * from sys.shards where table_name = 'blobs'");
        assertThat(response.rowCount(), is(2L));
        for (int i = 0; i<response.rowCount(); i++) {
View Full Code Here

            transportExecutor = SQLTransportExecutor.create(ClassLifecycleIntegrationTest.GLOBAL_CLUSTER);
            Setup setup = new Setup(transportExecutor);
            setup.groupBySetup();
            transportExecutor.exec(
                "create table quotes (id integer primary key, quote string) with(number_of_replicas=1)");
            BlobIndices blobIndices = GLOBAL_CLUSTER.getInstance(BlobIndices.class);
            Settings indexSettings = ImmutableSettings.builder()
                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5)
                    .build();
            blobIndices.createBlobTable("blobs", indexSettings);
            transportExecutor.ensureGreen();
            dataInitialized = true;
        }
    }
View Full Code Here

            bind(IndicesService.class).toInstance(indicesService);
            when(indicesService.indexServiceSafe(TEST_TABLE_NAME)).thenReturn(indexService);

            bind(Settings.class).toInstance(ImmutableSettings.EMPTY);

            BlobIndices blobIndices = new BlobIndices(
                    ImmutableSettings.EMPTY,
                    mock(TransportCreateIndexAction.class),
                    mock(TransportDeleteIndexAction.class),
                    mock(TransportUpdateSettingsAction.class),
                    indicesService,
View Full Code Here

        FileSystemUtils.deleteRecursively(globalBlobPath);
    }

    @Test
    public void testGlobalBlobPath() throws Exception {
        BlobIndices blobIndices = cluster().getInstance(BlobIndices.class, "node_0");
        BlobEnvironment blobEnvironment = cluster().getInstance(BlobEnvironment.class, "node_0");
        BlobEnvironment blobEnvironment2 = cluster().getInstance(BlobEnvironment.class, "node_1");
        assertThat(blobEnvironment.blobsPath().getAbsolutePath(), is(globalBlobPath.getAbsolutePath()));

        Settings indexSettings = ImmutableSettings.builder()
                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2)
                .build();
        blobIndices.createBlobTable("test", indexSettings).get();
        ensureGreen();
        assertTrue(blobEnvironment.shardLocation(new ShardId(".blob_test", 0)).exists()
         || blobEnvironment.shardLocation(new ShardId(".blob_test", 1)).exists());
        assertTrue(blobEnvironment2.shardLocation(new ShardId(".blob_test", 0)).exists()
                || blobEnvironment2.shardLocation(new ShardId(".blob_test", 1)).exists());

        blobIndices.dropBlobTable("test").get();
        assertFalse(blobEnvironment.indexLocation(new Index(".blob_test")).exists());
        assertFalse(blobEnvironment2.indexLocation(new Index(".blob_test")).exists());
    }
View Full Code Here

        assertFalse(blobEnvironment2.indexLocation(new Index(".blob_test")).exists());
    }

    @Test
    public void testPerTableBlobPath() throws Exception {
        BlobIndices blobIndices = cluster().getInstance(BlobIndices.class, "node_0");
        BlobEnvironment blobEnvironment = cluster().getInstance(BlobEnvironment.class, "node_0");
        BlobEnvironment blobEnvironment2 = cluster().getInstance(BlobEnvironment.class, "node_1");
        assertThat(blobEnvironment.blobsPath().getAbsolutePath(), is(globalBlobPath.getAbsolutePath()));

        File tempBlobPath = Files.createTempDirectory(null).toFile();
        Settings indexSettings = ImmutableSettings.builder()
                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2)
                .put(BlobIndices.SETTING_INDEX_BLOBS_PATH, tempBlobPath.getAbsolutePath())
                .build();
        blobIndices.createBlobTable("test", indexSettings).get();
        ensureGreen();
        assertTrue(blobEnvironment.shardLocation(new ShardId(".blob_test", 0), tempBlobPath).exists()
                || blobEnvironment.shardLocation(new ShardId(".blob_test", 1), tempBlobPath).exists());
        assertTrue(blobEnvironment2.shardLocation(new ShardId(".blob_test", 0), tempBlobPath).exists()
                || blobEnvironment2.shardLocation(new ShardId(".blob_test", 1), tempBlobPath).exists());

        blobIndices.createBlobTable("test2", indexSettings).get();
        ensureGreen();
        assertTrue(blobEnvironment.shardLocation(new ShardId(".blob_test2", 0), tempBlobPath).exists()
                || blobEnvironment.shardLocation(new ShardId(".blob_test2", 1), tempBlobPath).exists());
        assertTrue(blobEnvironment2.shardLocation(new ShardId(".blob_test2", 0), tempBlobPath).exists()
                || blobEnvironment2.shardLocation(new ShardId(".blob_test2", 1), tempBlobPath).exists());

        blobIndices.dropBlobTable("test").get();
        assertFalse(blobEnvironment.indexLocation(new Index(".blob_test"), tempBlobPath).exists());
        assertFalse(blobEnvironment2.indexLocation(new Index(".blob_test"), tempBlobPath).exists());

        // blobs path still exists because other index is using it
        assertTrue(tempBlobPath.exists());

        blobIndices.dropBlobTable("test2").get();
        assertFalse(blobEnvironment.indexLocation(new Index(".blob_test2"), tempBlobPath).exists());
        assertFalse(blobEnvironment2.indexLocation(new Index(".blob_test2"), tempBlobPath).exists());

        // no index using the blobs path anymore, should be deleted
        assertFalse(tempBlobPath.exists());

        blobIndices.createBlobTable("test", indexSettings).get();
        ensureGreen();

        File customFile = new File(tempBlobPath, "test_file");
        customFile.createNewFile();

        blobIndices.dropBlobTable("test").get();

        // blobs path still exists because a user defined file exists at the path
        assertTrue(tempBlobPath.exists());
        assertTrue(customFile.exists());
View Full Code Here

    public void setup() throws ExecutionException, InterruptedException {
        Iterable<HttpServerTransport> transports = cluster().getInstances(HttpServerTransport.class);
        Iterator<HttpServerTransport> httpTransports = transports.iterator();
        address = ((InetSocketTransportAddress) httpTransports.next().boundAddress().publishAddress()).address();
        address2 = ((InetSocketTransportAddress) httpTransports.next().boundAddress().publishAddress()).address();
        BlobIndices blobIndices = cluster().getInstance(BlobIndices.class);

        Settings indexSettings = ImmutableSettings.builder()
                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2)
                .build();
        blobIndices.createBlobTable("test", indexSettings).get();
        blobIndices.createBlobTable("test_blobs2", indexSettings).get();

        client().admin().indices().prepareCreate("test_no_blobs")
                .setSettings(
                        ImmutableSettings.builder()
                                .put("number_of_shards", 2)
View Full Code Here

        final int numberOfRelocations = 1;
        final int numberOfWriters = 2;

        final String node1 = cluster().startNode();

        BlobIndices blobIndices = cluster().getInstance(BlobIndices.class, node1);

        logger.trace("--> creating test index ...");
        Settings indexSettings = ImmutableSettings.builder()
                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
                .build();
        blobIndices.createBlobTable("test", indexSettings).get();

        logger.trace("--> starting [node2] ...");
        final String node2 = cluster().startNode();
        ensureGreen();

        final AtomicLong idGenerator = new AtomicLong();
        final AtomicLong indexCounter = new AtomicLong();
        final AtomicBoolean stop = new AtomicBoolean(false);
        Thread[] writers = new Thread[numberOfWriters];
        final CountDownLatch stopLatch = new CountDownLatch(writers.length);

        logger.trace("--> starting {} blob upload threads", writers.length);
        final List<String> uploadedDigests = Collections.synchronizedList(new ArrayList<String>(writers.length));
        for (int i = 0; i < writers.length; i++) {
            final int indexerId = i;
            writers[i] = new Thread() {
                @Override
                public void run() {
                    try {
                        logger.trace("**** starting blob upload thread {}", indexerId);
                        while (!stop.get()) {
                            long id = idGenerator.incrementAndGet();
                            String digest = uploadFile(cluster().client(node1), genFile(id));
                            uploadedDigests.add(digest);
                            indexCounter.incrementAndGet();
                        }
                        logger.trace("**** done indexing thread {}", indexerId);
                    } catch (Exception e) {
                        logger.warn("**** failed indexing thread {}", e, indexerId);
                    } finally {
                        stopLatch.countDown();
                    }
                }
            };
            writers[i].start();
        }

        logger.trace("--> waiting for 2 blobs to be uploaded ...");
        while (uploadedDigests.size() < 2) {
            Thread.sleep(10);
        }
        logger.trace("--> 2 blobs uploaded");

        // increase time between chunks in order to make sure that the upload is taking place while relocating
        timeBetweenChunks.set(10);
        stop.set(true);
        logger.trace("--> starting relocations...");
        for (int i = 0; i < numberOfRelocations; i++) {
            String fromNode = (i % 2 == 0) ? node1 : node2;
            String toNode = node1.equals(fromNode) ? node2 : node1;
            logger.trace("--> START relocate the shard from {} to {}", fromNode, toNode);
            cluster().client(node1).admin().cluster().prepareReroute()
                    .add(new MoveAllocationCommand(new ShardId(BlobIndices.fullIndexName("test"), 0), fromNode, toNode))
                    .execute().actionGet();
            ClusterHealthResponse clusterHealthResponse = cluster().client(node1).admin().cluster()
                    .prepareHealth()
                    .setWaitForEvents(Priority.LANGUID)
                    .setWaitForRelocatingShards(0)
                    .setTimeout(ACCEPTABLE_RELOCATION_TIME).execute().actionGet();

            assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
            clusterHealthResponse = cluster().client(node2).admin().cluster()
                    .prepareHealth()
                    .setWaitForEvents(Priority.LANGUID)
                    .setWaitForRelocatingShards(0)
                    .setTimeout(ACCEPTABLE_RELOCATION_TIME).execute().actionGet();
            assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
            logger.trace("--> DONE relocate the shard from {} to {}", fromNode, toNode);
        }
        logger.trace("--> done relocations");

        logger.trace("--> marking and waiting for upload threads to stop ...");
        timeBetweenChunks.set(0);
        stop.set(true);
        stopLatch.await(60, TimeUnit.SECONDS);
        logger.trace("--> uploading threads stopped");

        logger.trace("--> expected {} got {}", indexCounter.get(), uploadedDigests.size());
        assertEquals(indexCounter.get(), uploadedDigests.size());

        blobIndices = cluster().getInstance(BlobIndices.class, node2);
        for (String digest : uploadedDigests) {
            BlobShard blobShard = blobIndices.localBlobShard(BlobIndices.fullIndexName("test"), digest);
            long length = blobShard.blobContainer().getFile(digest).length();
            assertThat(length, greaterThanOrEqualTo(1L));
        }

    }
View Full Code Here

    private CollectingProjector getProjector(BlobContainer container,
                                             List<Input<?>> inputs,
                                             List<BlobCollectorExpression<?>> expressions,
                                             Input<Boolean> condition) throws Exception {
        CollectingProjector projector = new CollectingProjector();
        BlobShard blobShard = mock(BlobShard.class);
        when(blobShard.blobContainer()).thenReturn(container);

        BlobDocCollector collector = new BlobDocCollector(
                blobShard,
                inputs,
                expressions,
View Full Code Here

        logger.trace("--> expected {} got {}", indexCounter.get(), uploadedDigests.size());
        assertEquals(indexCounter.get(), uploadedDigests.size());

        blobIndices = cluster().getInstance(BlobIndices.class, node2);
        for (String digest : uploadedDigests) {
            BlobShard blobShard = blobIndices.localBlobShard(BlobIndices.fullIndexName("test"), digest);
            long length = blobShard.blobContainer().getFile(digest).length();
            assertThat(length, greaterThanOrEqualTo(1L));
        }

    }
View Full Code Here

TOP

Related Classes of io.crate.blob.v2.BlobIndexModule

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.