Package com.alimama.mdrill.hdfsDirectory

Examples of com.alimama.mdrill.hdfsDirectory.FileSystemDirectory$FileSystemIndexInput


  } else {
      moveToTrash(iconf, perm);
      fs.mkdirs(perm);
  }

  this.dir = new FileSystemDirectory(fs, perm, true, iconf);
  writer = new IndexWriter(dir, null,
          new KeepOnlyLastCommitDeletionPolicy(),
          MaxFieldLength.UNLIMITED);
//  writer.setMergeScheduler(new SerialMergeScheduler());//new ConcurrentMergeScheduler()
  writer.setMergeFactor(256);
View Full Code Here


    public void sync(Collection<String> names) throws IOException {
        Set<String> toSync = new HashSet<String>(names);
        Set<String> toSyncHdfs = new HashSet<String>(names);

       Configuration conf = getConf();
       FileSystemDirectory dir=null;
        for (String name : names)
        {
         if (this.links.containsKey(name)) {
            File file = this.links.get(name);
            this.fsync(file);
         }else if (this.hdfsLinks.containsKey(name)) {
            Path p = this.hdfsLinks.get(name);
            if(dir==null)
            {
              try {
                dir=new FileSystemDirectory(FileSystem.get(conf), p.getParent(), false, conf);
              } catch (IOException e) {
                SolrCore.log.error("hdfs",e);
              }
            }
            toSyncHdfs.add(name);

         }else{
           toSync.add(name);
         }
        }
       
        if(dir!=null)
        {
          dir.sync(toSyncHdfs);
        }
        super.sync(toSync);
      }
View Full Code Here

        } else {
          proxStream = null;
        }
      }else if(cfsDir instanceof FileSystemDirectory)
      {
        FileSystemDirectory dddir=(FileSystemDirectory)cfsDir;
        freqStream = new BlockBufferInput.KeyInput(cfsDir.openInput(filename, readBufferSize), cfsDir,filename,cfsDir.getP());
        if (fieldInfos.hasProx()) {
      String fname=IndexFileNames.segmentFileName(segment, IndexFileNames.PROX_EXTENSION);
          proxStream = BlockBufferInput.MaybeInstance(cfsDir.openInput(fname, readBufferSize),cfsDir,fname,cfsDir.getP());
        } else {
View Full Code Here

              {
               
                FSDirectory d=LinkFSDirectory.open(new File(s1));
                dirlist.add(d);
              }else{
                dirlist.add(new FileSystemDirectory( FileSystem.get(conf), new Path(s1.replaceAll("@hdfs@", "")), false, conf));
              }
              SolrCore.log.info(">>>>>LinkFSDirectory readOnlyOpen add links "+s1);
            }
          }
          br.close();
View Full Code Here

      {
        Configuration conf = getConf();
        Path p=this.hdfsLinks.get(name);
       
        try {
        FileSystemDirectory dir=new FileSystemDirectory(FileSystem.get(conf), p.getParent(), false, conf);
        return dir.fileModified(p.getName());
      } catch (IOException e) {
        SolrCore.log.error("hdfs",e);
      }
      }
     
View Full Code Here

      {
        Configuration conf = getConf();
        Path p=this.hdfsLinks.get(name);
       
        try {
        FileSystemDirectory dir=new FileSystemDirectory(FileSystem.get(conf), p.getParent(), false, conf);
        dir.touchFile(p.getName());
      } catch (IOException e) {
        SolrCore.log.error("hdfs",e);
      }
      }
      if(!this.links.containsKey(name))
View Full Code Here

      {
        Configuration conf = getConf();
        Path p=this.hdfsLinks.get(name);
       
        try {
        FileSystemDirectory dir=new FileSystemDirectory(FileSystem.get(conf), p.getParent(), false, conf);
        return dir.fileLength(p.getName());
      } catch (IOException e) {
        SolrCore.log.error("hdfs",e);
      }
      }
   
View Full Code Here

      {
        Configuration conf = getConf();
        Path p=this.hdfsLinks.get(name);
       
        try {
        FileSystemDirectory dir=new FileSystemDirectory(FileSystem.get(conf), p.getParent(), false, conf);
        return dir.createOutput(p.getName());
      } catch (IOException e) {
        SolrCore.log.error("hdfs",e);
      }
      }
     
View Full Code Here

            FileUtils.forceDelete(fd.getDirectory());
    }
   
    if(m1.d instanceof FileSystemDirectory)
    {
      FileSystemDirectory fd=(FileSystemDirectory)m1.d;
      FileSystem fs=FileSystem.get(conf);
      fs.delete(fd.directory,true);
    }
  }
View Full Code Here

    Configuration conf = context.getConfiguration();
    FileSystem fs = FileSystem.get(conf);
    Iterator<Text> iterator = values.iterator();
    while (iterator.hasNext()) {
      Text path = iterator.next();
      shardWriter.process(new FileSystemDirectory(fs, new Path(path
          .toString()), false, conf));
      context.write(key, path);
    }

  }
View Full Code Here

TOP

Related Classes of com.alimama.mdrill.hdfsDirectory.FileSystemDirectory$FileSystemIndexInput

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.