Examples of org.apache.hadoop.yarn.api.records.LocalResource

Package org.apache.hadoop.yarn.api.records

Examples of org.apache.hadoop.yarn.api.records.LocalResource

org.apache.hadoop.yarn.api.records.LocalResource

LocalResource represents a local resource required to run a container.

The NodeManager is responsible for localizing the resource prior to launching the container.

Applications can specify {@link LocalResourceType} and {@link LocalResourceVisibility}.
@see LocalResourceType @see LocalResourceVisibility @see ContainerLaunchContext @see ApplicationSubmissionContext @see ContainerManager#startContainer(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest)

    out.putNextEntry(new JarEntry(p.getName()));
    out.write(bytes);
    out.closeEntry();
    out.close();


    LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
    ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
        + ".jar")));
    ret.setSize(len);
    ret.setType(LocalResourceType.ARCHIVE);
    ret.setVisibility(vis);
    ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".jar"))
        .getModificationTime());
    return ret;
  }

View Full Code Here

    out.putNextEntry(new ZipEntry(p.getName()));
    out.write(bytes);
    out.closeEntry();
    out.close();


    LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
    ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
        + ".zip")));
    ret.setSize(len);
    ret.setType(LocalResourceType.ARCHIVE);
    ret.setVisibility(vis);
    ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".zip"))
        .getModificationTime());
    return ret;
  }

View Full Code Here

    LocalDirAllocator dirs =
      new LocalDirAllocator(TestFSDownload.class.getName());
    int size = 512;
    LocalResourceVisibility vis = LocalResourceVisibility.PUBLIC;
    Path path = new Path(basedir, "test-file");
    LocalResource rsrc = createFile(files, path, size, rand, vis);
    rsrcVis.put(rsrc, vis);
    Path destPath = dirs.getLocalPathForWrite(
        basedir.toString(), size, conf);
    destPath = new Path (destPath,
      Long.toString(uniqueNumberGenerator.incrementAndGet()));

View Full Code Here

      LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
      if (i%2 == 1) {
        vis = LocalResourceVisibility.APPLICATION;
      }
      Path p = new Path(basedir, "" + i);
      LocalResource rsrc = createFile(files, p, sizes[i], rand, vis);
      rsrcVis.put(rsrc, vis);
      Path destPath = dirs.getLocalPathForWrite(
          basedir.toString(), sizes[i], conf);
      destPath = new Path (destPath,
          Long.toString(uniqueNumberGenerator.incrementAndGet()));

View Full Code Here


    int size = rand.nextInt(512) + 512;
    LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;


    Path p = new Path(basedir, "" + 1);
    LocalResource rsrc = null;
    switch (fileType) {
    case TAR:
      rsrc = createTarFile(files, p, size, rand, vis);
      break;
    case JAR:
      rsrc = createJarFile(files, p, size, rand, vis);
      rsrc.setType(LocalResourceType.PATTERN);
      break;
    case ZIP:
      rsrc = createZipFile(files, p, size, rand, vis);
      break;
    case TGZ:

View Full Code Here

      if (i%2 == 1) {
        vis = LocalResourceVisibility.APPLICATION;
      }


      Path p = new Path(basedir, "dir" + i + ".jar");
      LocalResource rsrc = createJar(files, p, vis);
      rsrcVis.put(rsrc, vis);
      Path destPath = dirs.getLocalPathForWrite(
          basedir.toString(), conf);
      destPath = new Path (destPath,
          Long.toString(uniqueNumberGenerator.incrementAndGet()));

View Full Code Here

        new Path(destPath, Long.toString(uniqueNumberGenerator
            .incrementAndGet()));


    Path p = new Path(basedir, "dir" + 0 + ".jar");
    LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
    LocalResource rsrc = createJar(files, p, vis);
    FSDownload fsd =
        new FSDownload(files, UserGroupInformation.getCurrentUser(), conf,
            destPath, rsrc);
    Future<Path> rPath = singleThreadedExec.submit(fsd);
    singleThreadedExec.shutdown();

View Full Code Here

    Configuration conf = new Configuration();
    return RecordFactoryProvider.getRecordFactory(conf);
  }


  static LocalResource createResource() {
    LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
    assertTrue(ret instanceof LocalResourcePBImpl);
    ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(
      "hdfs://y.ak:8020/foo/bar")));
    ret.setSize(4344L);
    ret.setTimestamp(3141592653589793L);
    ret.setVisibility(LocalResourceVisibility.PUBLIC);
    return ret;
  }

View Full Code Here

      throws URISyntaxException {
    LocalizerHeartbeatResponse ret =
      recordFactory.newRecordInstance(LocalizerHeartbeatResponse.class);
    assertTrue(ret instanceof LocalizerHeartbeatResponsePBImpl);
    ret.setLocalizerAction(LocalizerAction.LIVE);
    LocalResource rsrc = createResource();
    ArrayList<ResourceLocalizationSpec> rsrcs =
      new ArrayList<ResourceLocalizationSpec>();
    ResourceLocalizationSpec resource =
      recordFactory.newRecordInstance(ResourceLocalizationSpec.class);
    resource.setResource(rsrc);

View Full Code Here

    }
  }


  public static LocalResource newLocalResource(URL url, LocalResourceType type,
      LocalResourceVisibility visibility, long size, long timestamp) {
    LocalResource resource =
      recordFactory.newRecordInstance(LocalResource.class);
    resource.setResource(url);
    resource.setType(type);
    resource.setVisibility(visibility);
    resource.setSize(size);
    resource.setTimestamp(timestamp);
    return resource;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.yarn.api.records.LocalResource

cascading.flow.tez.Hadoop2TezFlowStep

cascading.flow.tez.util.TezUtil

com.alibaba.jstorm.yarn.Util

com.cloudera.iterativereduce.Utils

com.cloudera.kitten.lua.LuaContainerLaunchParameters

com.hortonworks.simpleyarnapp.Client

com.yahoo.storm.yarn.Util

eu.stratosphere.yarn.ApplicationMaster

eu.stratosphere.yarn.Client

org.apache.flink.yarn.appMaster.ApplicationMaster

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.