Examples of ProxyOutputCommitter


Examples of com.datasalt.pangool.tuplemr.mapred.lib.output.ProxyOutputFormat.ProxyOutputCommitter

            this.context.getOutputFormatClass(), this.context.getConfiguration()));
      } catch(ClassNotFoundException e1) {
        throw new RuntimeException(e1);
      }

      ProxyOutputCommitter baseOutputCommitter = ((ProxyOutputCommitter) mainOutputFormat
          .getOutputCommitter(this.context));

      // The trick is to create a new Job for each output
      Configuration c = new Configuration(this.context.getConfiguration());
      Job job = new Job(c);

      Class<?> keyClass = getNamedOutputKeyClass(this.context, baseFileName);
      if(keyClass == null && getDefaultNamedOutputKeyClass(this.context) == null) {
        throw new InvalidNamedOutputException("No pre-configured named output for this name / no default named output format specified.");
      }
      job.setOutputKeyClass(keyClass == null ? getDefaultNamedOutputKeyClass(this.context) : keyClass);

      Class<?> valueClass = getNamedOutputValueClass(this.context, baseFileName);
      job.setOutputValueClass(valueClass == null ? getDefaultNamedOutputValueClass(this.context)
          : valueClass);

      // Check possible specific context for the output
      setSpecificNamedOutputContext(this.context.getConfiguration(), job, baseFileName);
      TaskAttemptContext taskContext;
      try {
        taskContext = TaskAttemptContextFactory.get(job.getConfiguration(),
            this.context.getTaskAttemptID());
      } catch(Exception e) {
        throw new IOException(e);
      }

      // First we change the output dir for the new OutputFormat that we will
      // create
      // We put it inside the main output work path -> in case the Job fails,
      // everything will be discarded

      taskContext.getConfiguration().set("mapred.output.dir",
          baseOutputCommitter.getBaseDir() + "/" + baseFileName);
      // This is for Hadoop 2.0 :
      taskContext.getConfiguration().set("mapreduce.output.fileoutputformat.outputdir",
          baseOutputCommitter.getBaseDir() + "/" + baseFileName);
      context.taskAttemptContext = taskContext;

      // Load the OutputFormat instance
      String outputFormatFile = getNamedOutputFormatInstanceFile(this.context, baseFileName);
      if(outputFormatFile == null) {
View Full Code Here

Examples of com.datasalt.pangool.tuplemr.mapred.lib.output.ProxyOutputFormat.ProxyOutputCommitter

            this.context.getOutputFormatClass(), this.context.getConfiguration()));
      } catch(ClassNotFoundException e1) {
        throw new RuntimeException(e1);
      }

      ProxyOutputCommitter baseOutputCommitter = ((ProxyOutputCommitter) mainOutputFormat
          .getOutputCommitter(this.context));

      // The trick is to create a new Job for each output
      Job job = new Job(this.context.getConfiguration());
      job.setOutputKeyClass(getNamedOutputKeyClass(this.context, baseFileName));
      job.setOutputValueClass(getNamedOutputValueClass(this.context, baseFileName));
      // Check possible specific context for the output
      setSpecificNamedOutputContext(this.context.getConfiguration(), job, baseFileName);
      TaskAttemptContext taskContext = new TaskAttemptContext(job.getConfiguration(),
          this.context.getTaskAttemptID());

      // First we change the output dir for the new OutputFormat that we will
      // create
      // We put it inside the main output work path -> in case the Job fails,
      // everything will be discarded
      taskContext.getConfiguration().set("mapred.output.dir",
          baseOutputCommitter.getBaseDir() + "/" + baseFileName);
      context.taskAttemptContext = taskContext;

      // Load the OutputFormat instance
      OutputFormat outputFormat = DCUtils.loadSerializedObjectInDC(
          context.taskAttemptContext.getConfiguration(), OutputFormat.class,
View Full Code Here

Examples of com.datasalt.pangool.tuplemr.mapred.lib.output.ProxyOutputFormat.ProxyOutputCommitter

            this.context.getOutputFormatClass(), this.context.getConfiguration()));
      } catch(ClassNotFoundException e1) {
        throw new RuntimeException(e1);
      }

      ProxyOutputCommitter baseOutputCommitter = ((ProxyOutputCommitter) mainOutputFormat
          .getOutputCommitter(this.context));

      // The trick is to create a new Job for each output
      Configuration c = new Configuration(this.context.getConfiguration());
      Job job = new Job(c);
      job.setOutputKeyClass(getNamedOutputKeyClass(this.context, baseFileName));
      job.setOutputValueClass(getNamedOutputValueClass(this.context, baseFileName));
      // Check possible specific context for the output
      setSpecificNamedOutputContext(this.context.getConfiguration(), job, baseFileName);
      TaskAttemptContext taskContext;
      try {
        taskContext = TaskAttemptContextFactory.get(job.getConfiguration(),
            this.context.getTaskAttemptID());
      } catch(Exception e) {
        throw new IOException(e);
      }

      // First we change the output dir for the new OutputFormat that we will
      // create
      // We put it inside the main output work path -> in case the Job fails,
      // everything will be discarded
     
      taskContext.getConfiguration().set("mapred.output.dir",
          baseOutputCommitter.getBaseDir() + "/" + baseFileName);
      // This is for Hadoop 2.0 :
      taskContext.getConfiguration().set("mapreduce.output.fileoutputformat.outputdir",
          baseOutputCommitter.getBaseDir() + "/" + baseFileName);
      context.taskAttemptContext = taskContext;

      // Load the OutputFormat instance
      OutputFormat outputFormat = InstancesDistributor.loadInstance(
          context.taskAttemptContext.getConfiguration(), OutputFormat.class,
View Full Code Here

Examples of com.datasalt.pangool.tuplemr.mapred.lib.output.ProxyOutputFormat.ProxyOutputCommitter

            this.context.getOutputFormatClass(), this.context.getConfiguration()));
      } catch(ClassNotFoundException e1) {
        throw new RuntimeException(e1);
      }

      ProxyOutputCommitter baseOutputCommitter = ((ProxyOutputCommitter) mainOutputFormat
          .getOutputCommitter(this.context));

      // The trick is to create a new Job for each output
      Job job = new Job(this.context.getConfiguration());
      job.setOutputKeyClass(getNamedOutputKeyClass(this.context, baseFileName));
      job.setOutputValueClass(getNamedOutputValueClass(this.context, baseFileName));
      // Check possible specific context for the output
      setSpecificNamedOutputContext(this.context.getConfiguration(), job, baseFileName);
      TaskAttemptContext taskContext;
      try {
        taskContext = TaskAttemptContextFactory.get(job.getConfiguration(),
            this.context.getTaskAttemptID());
      } catch(Exception e) {
        throw new IOException(e);
      }

      // First we change the output dir for the new OutputFormat that we will
      // create
      // We put it inside the main output work path -> in case the Job fails,
      // everything will be discarded
      taskContext.getConfiguration().set("mapred.output.dir",
          baseOutputCommitter.getBaseDir() + "/" + baseFileName);
      // This is for Hadoop 2.0 :
      taskContext.getConfiguration().set("mapreduce.output.fileoutputformat.outputdir",
          baseOutputCommitter.getBaseDir() + "/" + baseFileName);
      context.taskAttemptContext = taskContext;

      // Load the OutputFormat instance
      OutputFormat outputFormat = InstancesDistributor.loadInstance(
          context.taskAttemptContext.getConfiguration(), OutputFormat.class,
View Full Code Here

Examples of com.datasalt.pangool.tuplemr.mapred.lib.output.ProxyOutputFormat.ProxyOutputCommitter

            this.context.getOutputFormatClass(), this.context.getConfiguration()));
      } catch(ClassNotFoundException e1) {
        throw new RuntimeException(e1);
      }

      ProxyOutputCommitter baseOutputCommitter = ((ProxyOutputCommitter) mainOutputFormat
          .getOutputCommitter(this.context));

      // The trick is to create a new Job for each output
      Job job = new Job(this.context.getConfiguration());
      job.setOutputKeyClass(getNamedOutputKeyClass(this.context, baseFileName));
      job.setOutputValueClass(getNamedOutputValueClass(this.context, baseFileName));
      // Check possible specific context for the output
      setSpecificNamedOutputContext(this.context.getConfiguration(), job, baseFileName);
      TaskAttemptContext taskContext = new TaskAttemptContext(job.getConfiguration(),
          this.context.getTaskAttemptID());

      // First we change the output dir for the new OutputFormat that we will
      // create
      // We put it inside the main output work path -> in case the Job fails,
      // everything will be discarded
      taskContext.getConfiguration().set("mapred.output.dir",
          baseOutputCommitter.getBaseDir() + "/" + baseFileName);
      context.taskAttemptContext = taskContext;

      // Load the OutputFormat instance
      OutputFormat outputFormat = InstancesDistributor.loadInstance(
          context.taskAttemptContext.getConfiguration(), OutputFormat.class,
View Full Code Here

Examples of com.datasalt.pangool.tuplemr.mapred.lib.output.ProxyOutputFormat.ProxyOutputCommitter

            this.context.getOutputFormatClass(), this.context.getConfiguration()));
      } catch(ClassNotFoundException e1) {
        throw new RuntimeException(e1);
      }

      ProxyOutputCommitter baseOutputCommitter = ((ProxyOutputCommitter) mainOutputFormat
          .getOutputCommitter(this.context));

      // The trick is to create a new Job for each output
      Job job = new Job(this.context.getConfiguration());
      job.setOutputKeyClass(getNamedOutputKeyClass(this.context, baseFileName));
      job.setOutputValueClass(getNamedOutputValueClass(this.context, baseFileName));
      // Check possible specific context for the output
      setSpecificNamedOutputContext(this.context.getConfiguration(), job, baseFileName);
      TaskAttemptContext taskContext = new TaskAttemptContext(job.getConfiguration(),
          this.context.getTaskAttemptID());

      // First we change the output dir for the new OutputFormat that we will
      // create
      // We put it inside the main output work path -> in case the Job fails,
      // everything will be discarded
      taskContext.getConfiguration().set("mapred.output.dir",
          baseOutputCommitter.getBaseDir() + "/" + baseFileName);
      context.taskAttemptContext = taskContext;

      // Load the OutputFormat instance
      OutputFormat outputFormat = DCUtils.loadSerializedObjectInDC(
          context.taskAttemptContext.getConfiguration(), OutputFormat.class,
View Full Code Here

Examples of com.datasalt.pangool.tuplemr.mapred.lib.output.ProxyOutputFormat.ProxyOutputCommitter

            this.context.getOutputFormatClass(), this.context.getConfiguration()));
      } catch(ClassNotFoundException e1) {
        throw new RuntimeException(e1);
      }

      ProxyOutputCommitter baseOutputCommitter = ((ProxyOutputCommitter) mainOutputFormat
          .getOutputCommitter(this.context));

      // The trick is to create a new Job for each output
      Job job = new Job(this.context.getConfiguration());
      job.setOutputKeyClass(getNamedOutputKeyClass(this.context, baseFileName));
      job.setOutputValueClass(getNamedOutputValueClass(this.context, baseFileName));
      // Check possible specific context for the output
      setSpecificNamedOutputContext(this.context.getConfiguration(), job, baseFileName);
      TaskAttemptContext taskContext = new TaskAttemptContext(job.getConfiguration(),
          this.context.getTaskAttemptID());

      // First we change the output dir for the new OutputFormat that we will
      // create
      // We put it inside the main output work path -> in case the Job fails,
      // everything will be discarded
      taskContext.getConfiguration().set("mapred.output.dir",
          baseOutputCommitter.getBaseDir() + "/" + baseFileName);
      context.taskAttemptContext = taskContext;

      // Load the OutputFormat instance
      OutputFormat outputFormat = InstancesDistributor.loadInstance(
          context.taskAttemptContext.getConfiguration(), OutputFormat.class,
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.