Package org.apache.lucene.index

Examples of org.apache.lucene.index.TieredMergePolicy


    if (mp instanceof LogMergePolicy) {
      LogMergePolicy lmp = (LogMergePolicy) mp;
      lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
      lmp.setNoCFSRatio(1.0);
    } else if (mp instanceof TieredMergePolicy) {
      TieredMergePolicy tmp = (TieredMergePolicy) mp;
      tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
      tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
      tmp.setNoCFSRatio(1.0);
    }
    MergeScheduler ms = w.getConfig().getMergeScheduler();
    if (ms instanceof ConcurrentMergeScheduler) {
      // wtf... shouldnt it be even lower since its 1 by default?!?!
      ((ConcurrentMergeScheduler) ms).setMaxMergesAndThreads(3, 2);
View Full Code Here


    }
    return logmp;
  }

  public static TieredMergePolicy newTieredMergePolicy(Random r) {
    TieredMergePolicy tmp = new TieredMergePolicy();
    if (rarely(r)) {
      tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 2, 4));
      tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 2, 4));
    } else {
      tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 5, 50));
      tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 5, 50));
    }
    tmp.setMaxMergedSegmentMB(0.2 + r.nextDouble() * 2.0);
    tmp.setFloorSegmentMB(0.2 + r.nextDouble() * 2.0);
    tmp.setExpungeDeletesPctAllowed(0.0 + r.nextDouble() * 30.0);
    tmp.setSegmentsPerTier(_TestUtil.nextInt(r, 2, 20));
    tmp.setUseCompoundFile(r.nextBoolean());
    tmp.setNoCFSRatio(0.1 + r.nextDouble()*0.8);
    tmp.setReclaimDeletesWeight(r.nextDouble()*4);
    return tmp;
  }
View Full Code Here

    MergePolicy mp = w.getConfig().getMergePolicy();
    if (mp instanceof LogMergePolicy) {
      LogMergePolicy lmp = (LogMergePolicy) mp;
      lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
    } else if (mp instanceof TieredMergePolicy) {
      TieredMergePolicy tmp = (TieredMergePolicy) mp;
      tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
      tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
    }

    MergeScheduler ms = w.getConfig().getMergeScheduler();
    if (ms instanceof ConcurrentMergeScheduler) {
      ((ConcurrentMergeScheduler) ms).setMaxThreadCount(2);
View Full Code Here

      TableContext tableContext = TableContext.create(tableDescriptor);
      _fieldManager = tableContext.getFieldManager();
      Analyzer analyzer = _fieldManager.getAnalyzerForIndex();

      _conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, analyzer);
      TieredMergePolicy mergePolicy = (TieredMergePolicy) _conf.getMergePolicy();
      mergePolicy.setUseCompoundFile(false);

      _overFlowConf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, analyzer);
      _overFlowConf.setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES);

      if (_indexLocally) {
View Full Code Here

  static MergePolicy newSortingMergePolicy(Sorter sorter) {
    // create a MP with a low merge factor so that many merges happen
    MergePolicy mp;
    if (random().nextBoolean()) {
      TieredMergePolicy tmp = newTieredMergePolicy(random());
      final int numSegs = _TestUtil.nextInt(random(), 3, 5);
      tmp.setSegmentsPerTier(numSegs);
      tmp.setMaxMergeAtOnce(_TestUtil.nextInt(random(), 2, numSegs));
      mp = tmp;
    } else {
      LogMergePolicy lmp = newLogMergePolicy(random());
      lmp.setMergeFactor(_TestUtil.nextInt(random(), 3, 5));
      mp = lmp;
View Full Code Here

    if (mp instanceof LogMergePolicy) {
      LogMergePolicy lmp = (LogMergePolicy) mp;
      lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
      lmp.setNoCFSRatio(1.0);
    } else if (mp instanceof TieredMergePolicy) {
      TieredMergePolicy tmp = (TieredMergePolicy) mp;
      tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
      tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
      tmp.setNoCFSRatio(1.0);
    }
    MergeScheduler ms = w.getConfig().getMergeScheduler();
    if (ms instanceof ConcurrentMergeScheduler) {
      // wtf... shouldnt it be even lower since its 1 by default?!?!
      ((ConcurrentMergeScheduler) ms).setMaxMergesAndThreads(3, 2);
View Full Code Here

      sdp = new SnapshotDeletionPolicy(_tableContext.getIndexDeletionPolicy());
    }
    conf.setIndexDeletionPolicy(sdp);
    conf.setMergedSegmentWarmer(new FieldBasedWarmer(shardContext, _isClosed));

    TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy();
    mergePolicy.setUseCompoundFile(false);
    conf.setMergeScheduler(mergeScheduler.getMergeScheduler());

    DirectoryReferenceCounter referenceCounter = new DirectoryReferenceCounter(directory, gc, closer);
    // This directory allows for warm up by adding tracing ability.
    TraceableDirectory dir = new TraceableDirectory(referenceCounter);
View Full Code Here

  private static void createShard(Configuration configuration, int i, Path path, int totalShardCount)
      throws IOException {
    HdfsDirectory hdfsDirectory = new HdfsDirectory(configuration, path);
    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
    TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy();
    mergePolicy.setUseCompoundFile(false);
    IndexWriter indexWriter = new IndexWriter(hdfsDirectory, conf);

    Partitioner<IntWritable, IntWritable> partitioner = new HashPartitioner<IntWritable, IntWritable>();
    int partition = partitioner.getPartition(new IntWritable(i), null, totalShardCount);
    assertEquals(i, partition);
View Full Code Here

    boolean optimize = _blurTask.getOptimize();

    if (optimize) {
      context.setStatus("Starting Copy-Optimize Phase");
      IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, _analyzer);
      TieredMergePolicy policy = (TieredMergePolicy) conf.getMergePolicy();
      policy.setUseCompoundFile(false);
      long s = System.currentTimeMillis();
      IndexWriter writer = new IndexWriter(getBiggerBuffers(destDirectory), conf);
      writer.addIndexes(reader);
      writer.close();
      long e = System.currentTimeMillis();
View Full Code Here

    nullCheck(_directory);
    nullCheck(_analyzer);
    IndexWriterConfig config = new IndexWriterConfig(LUCENE_VERSION, _analyzer);
    config.setSimilarity(new FairSimilarity());
    config.setRAMBufferSizeMB(_blurTask.getRamBufferSizeMB());
    TieredMergePolicy mergePolicy = (TieredMergePolicy) config.getMergePolicy();
    mergePolicy.setUseCompoundFile(false);
    _writer = new IndexWriter(_directory, config);
  }
View Full Code Here

TOP

Related Classes of org.apache.lucene.index.TieredMergePolicy

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.