Examples of OutputJobInfo

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

295
296
297
298
299
300
301
302
303
304
305
    HCatWriterMapper.setWrittenRecordCount(0);

    FileInputFormat.setInputPaths(job, path);
    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(HCatOutputFormat.class);
    OutputJobInfo outputJobInfo = OutputJobInfo.create(dbName, tableName,
      partKeyMap);

    HCatOutputFormat.setOutput(job, outputJobInfo);
    HCatOutputFormat.setSchema(job, tblSchema);
    job.setMapOutputKeyClass(BytesWritable.class);
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

55
56
57
58
59
60
61
62
63
64
65
    super(config, sp);
  }

  @Override
  public WriterContext prepareWrite() throws HCatException {
    OutputJobInfo jobInfo = OutputJobInfo.create(we.getDbName(), we.getTableName(), we.getPartitionKVs());
    Job job;
    try {
      job = new Job(conf);
      HCatOutputFormat.setOutput(job, jobInfo);
      HCatOutputFormat.setSchema(job, HCatOutputFormat.getTableSchema(job));
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

207
208
209
210
211
212
213
214
215
216
217
218
219
        job.setOutputCommitter(HBaseBulkOutputCommitter.class);

        //manually create transaction
        RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf);
        try {
            OutputJobInfo outputJobInfo = OutputJobInfo.create("default", tableName, null);
            Transaction txn = rm.beginWriteTransaction(tableName, Arrays.asList(familyName));
            outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY,
                                                      HCatUtil.serialize(txn));
            job.set(HCatConstants.HCAT_KEY_OUTPUT_INFO,
                                       HCatUtil.serialize(outputJobInfo));
        } finally {
            rm.close();
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

381
382
383
384
385
386
387
388
389
390
391
        job.setInputFormatClass(TextInputFormat.class);
        TextInputFormat.setInputPaths(job, inputPath);


        job.setOutputFormatClass(HCatOutputFormat.class);
        OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName,tableName,null);
        HCatOutputFormat.setOutput(job,outputJobInfo);

        job.setMapOutputKeyClass(BytesWritable.class);
        job.setMapOutputValueClass(HCatRecord.class);
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

473
474
475
476
477
478
479
480
481
482
483
        job.setInputFormatClass(TextInputFormat.class);
        TextInputFormat.setInputPaths(job, inputPath);


        job.setOutputFormatClass(HCatOutputFormat.class);
        OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName,tableName,null);
        HCatOutputFormat.setOutput(job,outputJobInfo);

        job.setMapOutputKeyClass(BytesWritable.class);
        job.setMapOutputValueClass(HCatRecord.class);
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

545
546
547
548
549
550
551
552
553
554
555
            os.write(Bytes.toBytes(data[i] + "\n"));
            os.close();
        }

        Path workingDir = new Path(methodTestDir, "mr_abort");
        OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName,
                tableName, null);
        Job job = configureJob(testName,
                conf, workingDir, MapWriteAbortTransaction.class,
                outputJobInfo, inputPath);
        assertFalse(job.waitForCompletion(true));
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

148
149
150
151
152
153
154
155
156
157
158
159
160
    }

    public static class MapHCatWrite extends Mapper<LongWritable, Text, BytesWritable, HCatRecord> {
        @Override
        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            OutputJobInfo jobInfo = (OutputJobInfo)HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO));
            HCatRecord record = new DefaultHCatRecord(3);
            HCatSchema schema = jobInfo.getOutputSchema();
            String vals[] = value.toString().split(",");
            record.setInteger("key",schema,Integer.parseInt(vals[0]));
            for(int i=1;i<vals.length;i++) {
                String pair[] = vals[i].split(":");
                record.set(pair[0],schema,pair[1]);
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

142
143
144
145
146
147
148
149
150
151
152
153
154
        job.set(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, tableName);

        //manually create transaction
        RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf);
        try {
            OutputJobInfo outputJobInfo = OutputJobInfo.create("default", tableName, null);
            Transaction txn = rm.beginWriteTransaction(tableName, Arrays.asList(familyName));
            outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY,
                                                      HCatUtil.serialize(txn));
            job.set(HCatConstants.HCAT_KEY_OUTPUT_INFO,
                                       HCatUtil.serialize(outputJobInfo));
        } finally {
            rm.close();
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

224
225
226
227
228
229
230
231
232
233
234
            os.close();
        }

        //create job
        Path workingDir = new Path(methodTestDir, "mr_work");
        OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName,
                tableName, null);
        Job job = configureJob(testName, conf, workingDir, MapHCatWrite.class,
                outputJobInfo, inputPath);
        assertTrue(job.waitForCompletion(true));
View Full Code Here

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

302
303
304
305
306
307
308
309
310
311
312
            os.write(Bytes.toBytes(data[i] + "\n"));
            os.close();
        }

        Path workingDir = new Path(methodTestDir, "mr_abort");
        OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName,
                tableName, null);
        Job job = configureJob(testName, conf, workingDir, MapWriteAbortTransaction.class,
                outputJobInfo, inputPath);
        assertFalse(job.waitForCompletion(true));
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.