Package org.apache.hadoop.yarn.api.records

Examples of org.apache.hadoop.yarn.api.records.URL


    ContainerId cId = createContainerId();
    containerLaunchContext.setContainerId(cId);

    containerLaunchContext.setUser(user);

    URL localResourceUri =
        ConverterUtils.getYarnUrlFromPath(localFS
            .makeQualified(new Path(scriptFile.getAbsolutePath())));
    LocalResource localResource =
        recordFactory.newRecordInstance(LocalResource.class);
    localResource.setResource(localResourceUri);
View Full Code Here


  }

  private static Entry<String, LocalResource> getMockRsrc(Random r,
      LocalResourceVisibility vis) {
    String name = Long.toHexString(r.nextLong());
    URL url = BuilderUtils.newURL("file", null, 0, "/local" + vis + "/" + name);
    LocalResource rsrc =
        BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis,
            r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L);
    return new SimpleEntry<String, LocalResource>(name, rsrc);
  }
View Full Code Here

    LocalResource packageResource = Records.newRecord(LocalResource.class);
    FileSystem fs = FileSystem.get(conf);
    Path packageFile = new Path(conf.get("bsp.jar"));
    // FIXME there seems to be a problem with the converter utils and URL
    // transformation
    URL packageUrl = ConverterUtils.getYarnUrlFromPath(packageFile
        .makeQualified(fs.getUri(), fs.getWorkingDirectory()));
    LOG.info("PackageURL has been composed to " + packageUrl.toString());
    try {
      LOG.info("Reverting packageURL to path: "
          + ConverterUtils.getPathFromYarnURL(packageUrl));
    } catch (URISyntaxException e) {
      LOG.fatal("If you see this error the workarround does not work", e);
    }

    FileStatus fileStatus = fs.getFileStatus(packageFile);
    packageResource.setResource(packageUrl);
    packageResource.setSize(fileStatus.getLen());
    packageResource.setTimestamp(fileStatus.getModificationTime());
    packageResource.setType(LocalResourceType.ARCHIVE);
    packageResource.setVisibility(LocalResourceVisibility.APPLICATION);
    LOG.info("Package resource: " + packageResource.getResource());

    ctx.setLocalResources(Collections.singletonMap("package", packageResource));

    /*
     * TODO Package classpath seems not to work if you're in pseudo distributed
     * mode, because the resource must not be moved, it will never be unpacked.
     * So we will check if our jar file has the file:// prefix and put it into
     * the CP directly
     */
    String cp = "$CLASSPATH:./*:./package/*:./*:";
    if (packageUrl.getScheme() != null && packageUrl.getScheme().equals("file")) {
      cp += packageFile.makeQualified(fs.getUri(), fs.getWorkingDirectory())
          .toString() + ":";
      LOG.info("Localized file scheme detected, adjusting CP to: " + cp);
    }
    String[] cmds = {
View Full Code Here

public class TestConverterUtils {
 
  @Test
  public void testConvertUrlWithNoPort() throws URISyntaxException {
    Path expectedPath = new Path("hdfs://foo.com");
    URL url = ConverterUtils.getYarnUrlFromPath(expectedPath);
    Path actualPath = ConverterUtils.getPathFromYarnURL(url);
    assertEquals(expectedPath, actualPath);
  }
View Full Code Here

    ContainerLaunchContext containerLaunchContext =
        Records.newRecord(ContainerLaunchContext.class);
    // Construct the Container-id
    ContainerId cId = createContainerId();

    URL localResourceUri =
        ConverterUtils.getYarnUrlFromPath(localFS.makeQualified(new Path(
          localResourceDir.getAbsolutePath())));

    LocalResource localResource =
        LocalResource.newInstance(localResourceUri, LocalResourceType.FILE,
View Full Code Here

    resource.setVirtualCores(vCores);
    return resource;
  }

  public static URL newURL(String scheme, String host, int port, String file) {
    URL url = recordFactory.newRecordInstance(URL.class);
    url.setScheme(scheme);
    url.setHost(host);
    url.setPort(port);
    url.setFile(file);
    return url;
  }
View Full Code Here

      fileWriter.write("\nexec sleep 100");
    }
    fileWriter.close();

    // upload the script file so that the container can run it
    URL resource_alpha =
        ConverterUtils.getYarnUrlFromPath(localFS
            .makeQualified(new Path(scriptFile.getAbsolutePath())));
    LocalResource rsrc_alpha =
        recordFactory.newRecordInstance(LocalResource.class);
    rsrc_alpha.setResource(resource_alpha);
View Full Code Here

    ContainerLaunchContext containerLaunchContext =
        recordFactory.newRecordInstance(ContainerLaunchContext.class);
    int port = 12345;

    // upload the script file so that the container can run it
    URL resource_alpha =
        ConverterUtils.getYarnUrlFromPath(localFS
            .makeQualified(new Path(scriptFile.getAbsolutePath())));
    LocalResource rsrc_alpha =
        recordFactory.newRecordInstance(LocalResource.class);
    rsrc_alpha.setResource(resource_alpha);
View Full Code Here

    ContainerId cId = createContainerId(0);

    // ////// Construct the container-spec.
    ContainerLaunchContext containerLaunchContext =
        recordFactory.newRecordInstance(ContainerLaunchContext.class);
    URL resource_alpha =
        ConverterUtils.getYarnUrlFromPath(localFS
            .makeQualified(new Path(file.getAbsolutePath())));
    LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);   
    rsrc_alpha.setResource(resource_alpha);
    rsrc_alpha.setSize(-1);
View Full Code Here

    fileWriter.close();

    ContainerLaunchContext containerLaunchContext =
        recordFactory.newRecordInstance(ContainerLaunchContext.class);

    URL resource_alpha =
        ConverterUtils.getYarnUrlFromPath(localFS
            .makeQualified(new Path(scriptFile.getAbsolutePath())));
    LocalResource rsrc_alpha =
        recordFactory.newRecordInstance(LocalResource.class);
    rsrc_alpha.setResource(resource_alpha);
View Full Code Here

TOP

Related Classes of org.apache.hadoop.yarn.api.records.URL

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.