Package cascading.tap

Examples of cascading.tap.TapException


        // Copy Solr core directory into HDFS.
        try {
            FileSystem fs = hdfsSolrCoreDir.getFileSystem(conf);
            fs.copyFromLocalFile(new Path(_solrCoreDir.getAbsolutePath()), hdfsSolrCoreDir);
        } catch (IOException e) {
            throw new TapException("Can't copy Solr core directory into HDFS", e);
        }

        conf.setOutputKeyClass(Tuple.class);
        conf.setOutputValueClass(Tuple.class);
        conf.setOutputFormat(SolrOutputFormat.class);

        try {
            conf.set(SolrOutputFormat.SINK_FIELDS_KEY, HadoopUtil.serializeBase64(getSinkFields(), conf));
        } catch (IOException e) {
            throw new TapException("Can't serialize sink fields", e);
        }

        conf.set(SolrOutputFormat.SOLR_CORE_PATH_KEY, hdfsSolrCoreDir.toString());
        conf.setInt(SolrOutputFormat.MAX_SEGMENTS_KEY, _maxSegments);
        conf.set(SolrOutputFormat.DATA_DIR_PROPERTY_NAME_KEY, _dataDirPropertyName);
View Full Code Here


        conf.set(SolrOutputFormat.DATA_DIR_PROPERTY_NAME_KEY, _dataDirPropertyName);
    }

    @Override
    public boolean source(FlowProcess<JobConf> conf, SourceCall<Object[], RecordReader<Tuple, Tuple>> sourceCall) throws IOException {
        throw new TapException("SolrScheme can only be used as a sink, not a source");
    }
View Full Code Here

   
    public static void validate(File solrCoreDir, String dataDirPropertyName, Fields schemeFields) throws IOException {
       
        // Verify solrHomeDir exists
        if (!solrCoreDir.exists() || !solrCoreDir.isDirectory()) {
            throw new TapException("Solr core directory doesn't exist: " + solrCoreDir);
        }
       
        File tmpSolrHome = makeTempSolrHome(solrCoreDir);
       
        // Set up a temp location for Solr home, where we're write out a synthetic solr.xml
        // that references the core directory.
        String coreName = solrCoreDir.getName();
        String corePath = solrCoreDir.getAbsolutePath();
        String solrXmlContent = String.format("<solr><cores><core name=\"%s\" instanceDir=\"%s\"></core></cores></solr>",
                                              coreName, corePath);
        File solrXmlFile = new File(tmpSolrHome, "solr.xml");
        FileUtils.write(solrXmlFile, solrXmlContent);
       
        // Set up a temp location for data, so when we instantiate the coreContainer,
        // we don't pollute the solr home with a /data sub-dir.
        String tmpFolder = System.getProperty("java.io.tmpdir");
        File tmpDataDir = new File(tmpFolder, UUID.randomUUID().toString());
        tmpDataDir.mkdir();
       
        System.setProperty(dataDirPropertyName, tmpDataDir.getAbsolutePath());
        System.setProperty("enable.special-handlers", "false"); // All we need is the update request handler
        System.setProperty("enable.cache-warming", "false"); // We certainly don't need to warm the cache
       
        CoreContainer coreContainer = new CoreContainer(tmpSolrHome.getAbsolutePath());
       
        try {
            coreContainer.load();
            Collection<SolrCore> cores = coreContainer.getCores();
            SolrCore core = null;
           
            if (cores.size() == 0) {
                throw new TapException("No Solr cores are available");
            } else if (cores.size() > 1) {
                throw new TapException("Only one Solr core is supported");
            } else {
                core = cores.iterator().next();
            }

            IndexSchema schema = core.getLatestSchema();
            Map<String, SchemaField> solrFields = schema.getFields();
            Set<String> schemeFieldnames = new HashSet<String>();

            for (int i = 0; i < schemeFields.size(); i++) {
                String fieldName = schemeFields.get(i).toString();
                if (!solrFields.containsKey(fieldName)) {
                    throw new TapException("Sink field name doesn't exist in Solr schema: " + fieldName);
                }
               
                schemeFieldnames.add(fieldName);
            }

            for (String solrFieldname : solrFields.keySet()) {
                SchemaField solrField = solrFields.get(solrFieldname);
                if (solrField.isRequired() && !schemeFieldnames.contains(solrFieldname)) {
                    throw new TapException("No sink field name for required Solr field: " + solrFieldname);
                }
            }
        } finally {
            if (coreContainer != null) {
                coreContainer.shutdown();
View Full Code Here

        return false;
    }
   
    @Override
    public void sourceConfInit(FlowProcess<Properties> flowProcess, Tap<Properties, InputStream, OutputStream> tap, Properties conf) {
        throw new TapException("SolrScheme can only be used as a sink, not a source");
    }
View Full Code Here

        throw new TapException("SolrScheme can only be used as a sink, not a source");
    }

    @Override
    public boolean source(FlowProcess<Properties> conf, SourceCall<Void, InputStream> sourceCall) throws IOException {
        throw new TapException("SolrScheme can only be used as a sink, not a source");
    }
View Full Code Here

    }

    @Override
    public void sinkPrepare(FlowProcess<Properties> flowProcess, SinkCall<SolrCollector, OutputStream> sinkCall) throws IOException {
        if (!(sinkCall.getOutput() instanceof DirectoryFileOutputStream)) {
            throw new TapException("SolrScheme can only be used with a DirectoryTap in local mode");
        }
       
        DirectoryFileOutputStream os = (DirectoryFileOutputStream)sinkCall.getOutput();
        String path = os.asDirectory();
View Full Code Here

      getScheme().sourceConfInit(process, this, conf);
      makeLocal(conf, getQualifiedPath(conf), "forcing job to local mode, via source: ");
      TupleSerialization.setSerializations(conf);
    } catch (IOException e) {
      throw new TapException(e);
    }
  }
View Full Code Here

  }

  @Override
  public void sinkConfInit(FlowProcess<JobConf> process, JobConf conf) {
    if (_options.attrs != null && _options.attrs.length > 0) {
      throw new TapException("can't declare attributes in a sink");
    }

    super.sinkConfInit(process, conf);
  }
View Full Code Here

        Tap<JobConf, RecordReader, OutputCollector> tap, JobConf conf) {
      Pail p;
      try {
        p = new Pail(_pailRoot); //make sure it exists
      } catch (IOException e) {
        throw new TapException(e);
      }
      conf.setInputFormat(p.getFormat().getInputFormatClass());
      PailFormatFactory.setPailPathLister(conf, _options.lister);
    }
View Full Code Here

      conf.setOutputFormat(PailOutputFormat.class);
      Utils.setObject(conf, PailOutputFormat.SPEC_ARG, getSpec());
      try {
        Pail.create(getFileSystem(conf), _pailRoot, getSpec(), true);
      } catch (IOException e) {
        throw new TapException(e);
      }
    }
View Full Code Here

TOP

Related Classes of cascading.tap.TapException

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.