Package org.apache.flink.configuration

Examples of org.apache.flink.configuration.Configuration

96
97
98
99
100
101
102
103
104
105
    compareResultsByLinesInMemory(result, resultPath);
  }

  @Parameterized.Parameters
  public static Collection<Object[]> getConfigurations() {
    Configuration config = new Configuration();
    config.setInteger("ChainedMapperNepheleITCase#NoSubtasks", DOP);
    config.setInteger("ChainedMapperNepheleITCase#MaxIterations", 2);
    return toParameterList(config);
  }
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
      headConfig.setOutputSerializer(serializer);
      headConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
      headConfig.setOutputComparator(comparator, 0);

      // final output
      TaskConfig headFinalOutConfig = new TaskConfig(new Configuration());
      headFinalOutConfig.setOutputSerializer(serializer);
      headFinalOutConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      headConfig.setIterationHeadFinalOutputConfig(headFinalOutConfig);

      // the sync
      headConfig.setIterationHeadIndexOfSyncOutput(2);

      // driver
      headConfig.setDriver(CollectorMapDriver.class);
      headConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      headConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));

      // back channel
      headConfig.setRelativeBackChannelMemory(1.0);
    }

    // - tail ------------------------------------------------------------------------------------------------------
    AbstractJobVertex tail = JobGraphUtils.createTask(IterationTailPactTask.class, "Chained Iteration Tail", jobGraph, numSubTasks);
    TaskConfig tailConfig = new TaskConfig(tail.getConfiguration());
    {
      tailConfig.setIterationId(ITERATION_ID);

      // inputs and driver
      tailConfig.addInputToGroup(0);
      tailConfig.setInputSerializer(serializer, 0);

      // output
      tailConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      tailConfig.setOutputSerializer(serializer);

      // the driver
      tailConfig.setDriver(GroupReduceDriver.class);
      tailConfig.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
      tailConfig.setDriverComparator(comparator, 0);
      tailConfig.setStubWrapper(new UserCodeClassWrapper<DummyReducer>(DummyReducer.class));

      // chained mapper
      TaskConfig chainedMapperConfig = new TaskConfig(new Configuration());
      chainedMapperConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      chainedMapperConfig.setStubWrapper(new UserCodeClassWrapper<IncrementCoordinatesMapper>(
        IncrementCoordinatesMapper.class));

      chainedMapperConfig.setInputLocalStrategy(0, LocalStrategy.NONE);
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
192
   */
  @Before
  public void writeTuples() throws IOException {
    this.tempFile = File.createTempFile("SerializedInputFormat", null);
    this.tempFile.deleteOnExit();
    Configuration configuration = new Configuration();
    configuration.setLong(BinaryOutputFormat.BLOCK_SIZE_PARAMETER_KEY, this.blockSize);
    if (this.degreeOfParallelism == 1) {
      SerializedOutputFormat output =
        FormatUtil.openOutput(SerializedOutputFormat.class, this.tempFile.toURI().toString(),
          configuration);
      for (int index = 0; index < this.numberOfTuples; index++) {
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
      InputFormat<?, ?> format = null;
      String inFormatDescription = "<unknown>";
     
      try {
        format = getPactContract().getFormatWrapper().getUserCodeObject();
        Configuration config = getPactContract().getParameters();
        format.configure(config);
      }
      catch (Throwable t) {
        if (PactCompiler.LOG.isWarnEnabled()) {
          PactCompiler.LOG.warn("Could not instantiate InputFormat to obtain statistics."
View Full Code Here
229
230
231
232
233
234
235
236
237
238
239
240
      Assert.assertEquals(expectedLength, files[fileIndex].length());
    }
  }

  protected SerializedInputFormat<Record> createInputFormat() {
    Configuration configuration = new Configuration();
    configuration.setLong(BinaryInputFormat.BLOCK_SIZE_PARAMETER_KEY, this.blockSize);

    final SerializedInputFormat<Record> inputFormat = new SerializedInputFormat<Record>();
    inputFormat.setFilePath(this.tempFile.toURI().toString());
   
    inputFormat.configure(configuration);
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
    for(int i = 0; i < blockSize * numBlocks; i++) {
      fileOutputStream.write(new byte[]{1});
    }
    fileOutputStream.close();

    final Configuration config = new Configuration();
    config.setLong(BinaryInputFormat.BLOCK_SIZE_PARAMETER_KEY, blockSize);
   
    final BinaryInputFormat<Record> inputFormat = new MyBinaryInputFormat();
    inputFormat.setFilePath(tempFile.toURI().toString());
   
    inputFormat.configure(config);
View Full Code Here
610
611
612
613
614
615
616
617
618
619
620
621
622
      task.setEnvironment(env);
     
      // register the task with the network stack and profilers
      this.channelManager.register(task);
     
      final Configuration jobConfig = tdd.getJobConfiguration();

      boolean enableProfiling = this.profiler != null && jobConfig.getBoolean(ProfilingUtils.PROFILE_JOB_KEY, true);

      // Register environment, input, and output gates for profiling
      if (enableProfiling) {
        task.registerProfiler(this.profiler, jobConfig);
      }
View Full Code Here
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
    // First, try to load global configuration
    GlobalConfiguration.loadConfiguration(configDir);
    if(tempDirVal != null // the YARN TM runner has set a value for the temp dir
        // the configuration does not contain a temp directory
        && GlobalConfiguration.getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, null) == null) {
      Configuration c = GlobalConfiguration.getConfiguration();
      c.setString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, tempDirVal);
      LOG.info("Setting temporary directory to "+tempDirVal);
      GlobalConfiguration.includeConfiguration(c);
    }
   
    // print some startup environment info, like user, code revision, etc
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
76
          linksInput.groupBy(0).reduceGroup(new BuildOutgoingEdgeList());
     
      // set iterative data set
      IterativeDataSet<Tuple2<Long, Double>> iteration = pagesWithRanks.iterate(10);
     
      Configuration cfg = new Configuration();
      cfg.setString(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND);
     
      DataSet<Tuple2<Long, Double>> newRanks = iteration
          // join pages with outgoing edges and distribute rank
          .join(adjacencyListInput).where(0).equalTo(0).withParameters(cfg)
          .flatMap(new JoinVertexWithEdgesMatch())
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
  public void initKeys() {
    final String accessKey = System.getenv("AK");
    final String secretKey = System.getenv("SK");
   
    if (accessKey != null || secretKey != null) {
      Configuration conf = new Configuration();
      if (accessKey != null) {
        conf.setString(S3FileSystem.S3_ACCESS_KEY_KEY, accessKey);
      }
      if (secretKey != null) {
        conf.setString(S3FileSystem.S3_SECRET_KEY_KEY, secretKey);
      }
      GlobalConfiguration.includeConfiguration(conf);
    }
  }
View Full Code Here
TOP

Related Classes of org.apache.flink.configuration.Configuration

  • org.apache.flink.api.avro.AvroExternalJarProgramITCase
  • org.apache.flink.api.common.io.BinaryInputFormatTest
  • org.apache.flink.api.common.io.DelimitedInputFormatSamplingTest
  • org.apache.flink.api.common.io.DelimitedInputFormatTest
  • org.apache.flink.api.common.io.FileInputFormatTest
  • org.apache.flink.api.common.io.FileOutputFormatTest
  • org.apache.flink.api.common.io.FormatUtil
  • org.apache.flink.api.common.io.GenericCsvInputFormatTest
  • org.apache.flink.api.common.io.SequentialFormatTest
  • org.apache.flink.api.io.avro.AvroRecordInputFormatTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.