Package org.apache.hcatalog.data.transfer

Examples of org.apache.hcatalog.data.transfer.ReaderContext

62
63
64
65
66
67
68
69
70
71
72
73
74
75
    public ReaderContext prepareRead() throws HCatException {
        try {
            Job job = new Job(conf);
            HCatInputFormat hcif = HCatInputFormat.setInput(
                job, re.getDbName(), re.getTableName()).setFilter(re.getFilterString());
            ReaderContext cntxt = new ReaderContext();
            cntxt.setInputSplits(hcif.getSplits(
                HCatHadoopShims.Instance.get().createJobContext(job.getConfiguration(), null)));
            cntxt.setConf(job.getConfiguration());
            return cntxt;
        } catch (IOException e) {
            throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
        } catch (InterruptedException e) {
            throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
        ois.close();

        runsInSlave(cntxt);
        commit(map, true, cntxt);

        ReaderContext readCntxt = runsInMaster(map, false);

        File readCntxtFile = File.createTempFile("hcat-read", "temp");
        readCntxtFile.deleteOnExit();
        oos = new ObjectOutputStream(new FileOutputStream(readCntxtFile));
        oos.writeObject(readCntxt);
        oos.flush();
        oos.close();

        ois = new ObjectInputStream(new FileInputStream(readCntxtFile));
        readCntxt = (ReaderContext) ois.readObject();
        ois.close();

        for (InputSplit split : readCntxt.getSplits()) {
            runsInSlave(split, readCntxt.getConf());
        }
    }
View Full Code Here
111
112
113
114
115
116
117
118
    private ReaderContext runsInMaster(Map<String, String> config, boolean bogus)
        throws HCatException {
        ReadEntity entity = new ReadEntity.Builder().withTable("mytbl").build();
        HCatReader reader = DataTransferFactory.getHCatReader(entity, config);
        ReaderContext cntxt = reader.prepareRead();
        return cntxt;
    }
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
public class DataReaderSlave {

    public static void main(String[] args) throws IOException, ClassNotFoundException {

        ObjectInputStream ois = new ObjectInputStream(new FileInputStream(new File(args[0])));
        ReaderContext cntxt = (ReaderContext) ois.readObject();
        ois.close();

        String[] inpSlitsToRead = args[1].split(",");
        List<InputSplit> splits = cntxt.getSplits();

        for (int i = 0; i < inpSlitsToRead.length; i++) {
            InputSplit split = splits.get(Integer.parseInt(inpSlitsToRead[i]));
            HCatReader reader = DataTransferFactory.getHCatReader(split, cntxt.getConf());
            Iterator<HCatRecord> itr = reader.read();
            File f = new File(args[2] + "-" + i);
            f.delete();
            BufferedWriter outFile = new BufferedWriter(new FileWriter(f));
            while (itr.hasNext()) {
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
        for (Entry<Object, Object> kv : externalConfigs.entrySet()) {
            config.put((String) kv.getKey(), (String) kv.getValue());
        }

        // This piece of code runs in master node and gets necessary context.
        ReaderContext context = runsInMaster(config);

        ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(new File(args[1])));
        oos.writeObject(context);
        oos.flush();
        oos.close();
View Full Code Here
63
64
65
66
67
68
69
70
    private static ReaderContext runsInMaster(Map<String, String> config) throws HCatException {

        ReadEntity.Builder builder = new ReadEntity.Builder();
        ReadEntity entity = builder.withTable(config.get("table")).build();
        HCatReader reader = DataTransferFactory.getHCatReader(entity, config);
        ReaderContext cntxt = reader.prepareRead();
        return cntxt;
    }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
    try {
      Job job = new Job(conf);
      InputJobInfo jobInfo = InputJobInfo.create(re.getDbName(), re.getTableName(), re.getFilterString());
      HCatInputFormat.setInput(job, jobInfo);
      HCatInputFormat hcif = new HCatInputFormat();
      ReaderContext cntxt = new ReaderContext();
      cntxt.setInputSplits(hcif.getSplits(new JobContext(job.getConfiguration(), null)));
      cntxt.setConf(job.getConfiguration());
      return cntxt;
    } catch (IOException e) {
      throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
    } catch (InterruptedException e) {
      throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED,e);
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
    ois.close();
   
    runsInSlave(cntxt);
    commit(map, true, cntxt);
   
    ReaderContext readCntxt = runsInMaster(map, false);
   
    File readCntxtFile = File.createTempFile("hcat-read", "temp");
    readCntxtFile.deleteOnExit();
    oos = new ObjectOutputStream(new FileOutputStream(readCntxtFile));
    oos.writeObject(readCntxt);
    oos.flush();
    oos.close();
   
    ois = new ObjectInputStream(new FileInputStream(readCntxtFile));
    readCntxt = (ReaderContext) ois.readObject();
    ois.close();
   
   
    for(InputSplit split : readCntxt.getSplits()){
      runsInSlave(split, readCntxt.getConf());     
    }
  }
View Full Code Here
117
118
119
120
121
122
123
124
  private ReaderContext runsInMaster(Map<String,String> config, boolean bogus) throws HCatException {

    ReadEntity.Builder builder = new ReadEntity.Builder();
    ReadEntity entity = builder.withTable("mytbl").build();
    HCatReader reader = DataTransferFactory.getHCatReader(entity, config);
    ReaderContext cntxt = reader.prepareRead();
    return cntxt;
  }
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
public class DataReaderSlave {

  public static void main(String[] args) throws IOException, ClassNotFoundException {
   
    ObjectInputStream ois = new ObjectInputStream(new FileInputStream(new File(args[0])));
    ReaderContext cntxt = (ReaderContext) ois.readObject();
    ois.close();
   
    String[] inpSlitsToRead = args[1].split(",");
    List<InputSplit> splits = cntxt.getSplits();
   
    for (int i = 0; i < inpSlitsToRead.length; i++){
      InputSplit split = splits.get(Integer.parseInt(inpSlitsToRead[i]));
      HCatReader reader = DataTransferFactory.getHCatReader(split, cntxt.getConf());
      Iterator<HCatRecord> itr = reader.read();
      File f = new File(args[2]+"-"+i);
      f.delete();
      BufferedWriter outFile = new BufferedWriter(new FileWriter(f));
      while(itr.hasNext()){
View Full Code Here
TOP

Related Classes of org.apache.hcatalog.data.transfer.ReaderContext

  • org.apache.hadoop.conf.Configuration
  • org.apache.hcatalog.data.TestReaderWriter
  • org.apache.hcatalog.data.transfer.impl.HCatInputFormatReader
  • org.apache.hcatalog.mapreduce.HCatSplit
  • org.apache.hcatalog.utils.DataReaderMaster
  • org.apache.hcatalog.utils.DataReaderSlave
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.