Package org.kitesdk.morphline.api

Examples of org.kitesdk.morphline.api.Record


    for (int i = 0; i < 1; i++) {
     
      for (String file : files) {
        File f = new File(file);
        byte[] body = Files.toByteArray(f);
        Record event = new Record();
        //event.put(Fields.ID, docId++);
        event.getFields().put(Fields.ATTACHMENT_BODY, new ByteArrayInputStream(body));
        event.getFields().put(Fields.ATTACHMENT_NAME, f.getName());
        event.getFields().put(Fields.BASE_ID, f.getName());       
        collector.reset();
        load(event);
        Integer count = expectedRecords.get(file);
        if (count != null) {
          numDocs += count;
        } else {
          numDocs++;
        }
        assertEquals("unexpected results in " + file, numDocs, queryResultSetSize("*:*"));
        Map<String, Object> expectedContents = expectedRecordContents.get(file);
        if (expectedContents != null) {
          Record actual = collector.getFirstRecord();
          for (Map.Entry<String, Object> entry : expectedContents.entrySet()) {
            if (entry.getValue() == NON_EMPTY_FIELD) {
              assertNotNull(entry.getKey());
              assertTrue(actual.getFirstValue(entry.getKey()).toString().length() > 0);
            } else if (entry.getValue() == null) {
              assertEquals("key:" + entry.getKey(), 0, actual.get(entry.getKey()).size());
            } else {
              assertEquals("key:" + entry.getKey(), Arrays.asList(entry.getValue()), actual.get(entry.getKey()));
            }
          }
        }
      }
    }
View Full Code Here


      this.valueField = getConfigs().getString(config, ReadSequenceFileBuilder.CONFIG_VALUE_FIELD, Fields.ATTACHMENT_BODY);
    }

    @Override
    protected boolean doProcess(Record inputRecord) {
      Record outputRecord = inputRecord.copy();

      // change key
      Text myTextKey = (Text)inputRecord.getFirstValue(this.keyField);
      outputRecord.replaceValues(this.keyField, MyWritable.keyStr(myTextKey));
      // change value
      MyWritable myWritableValue = (MyWritable)inputRecord.getFirstValue(this.valueField);
      outputRecord.replaceValues(this.valueField, MyWritable.valueStr(myWritableValue));
      return super.doProcess(outputRecord);
    }
View Full Code Here

   
    @Override
    protected boolean doProcess(Record inputRecord) {
      JsonNode datum = (JsonNode) inputRecord.getFirstValue(Fields.ATTACHMENT_BODY);
      Preconditions.checkNotNull(datum);
      Record outputRecord = inputRecord.copy();
     
      for (Map.Entry<String, Collection<String>> entry : stepMap.entrySet()) {
        String fieldName = entry.getKey();
        List<String> steps = (List<String>) entry.getValue();
        extractPath(datum, fieldName, steps, outputRecord, 0);
View Full Code Here

    waitForRecoveriesToFinish(false);
   
    createAlias("aliascollection", "collection1");
   
    morphline = parse("test-morphlines" + File.separator + "loadSolrBasic", "aliascollection");
    Record record = new Record();
    record.put(Fields.ID, "id0-innsbruck");
    record.put("text", "mytext");
    record.put("user_screen_name", "foo");
    record.put("first_name", "Nadja"); // will be sanitized
    startSession();
    assertEquals(1, collector.getNumStartEvents());
    Notifications.notifyBeginTransaction(morphline);
    assertTrue(morphline.process(record));
   
    record = new Record();
    record.put(Fields.ID, "id1-innsbruck");
    record.put("text", "mytext1");
    record.put("user_screen_name", "foo1");
    record.put("first_name", "Nadja1"); // will be sanitized
    assertTrue(morphline.process(record));
   
    Record expected = new Record();
    expected.put(Fields.ID, "id0-innsbruck");
    expected.put("text", "mytext");
    expected.put("user_screen_name", "foo");
    Iterator<Record> citer = collector.getRecords().iterator();
    assertEquals(expected, citer.next());
   
    Record expected2 = new Record();
    expected2.put(Fields.ID, "id1-innsbruck");
    expected2.put("text", "mytext1");
    expected2.put("user_screen_name", "foo1");
    assertEquals(expected2, citer.next());
   
    assertFalse(citer.hasNext());
   
    commit();
   
    QueryResponse rsp = cloudClient.query(new SolrQuery("*:*").setRows(100000).addSort(Fields.ID, SolrQuery.ORDER.asc));
    //System.out.println(rsp);
    Iterator<SolrDocument> iter = rsp.getResults().iterator();
    assertEquals(expected.getFields(), next(iter));
    assertEquals(expected2.getFields(), next(iter));
    assertFalse(iter.hasNext());
   
    Notifications.notifyRollbackTransaction(morphline);
    Notifications.notifyShutdown(morphline);
   
View Full Code Here

      validateArguments();
    }

    @Override
    protected boolean doProcess(Record inputRecord, InputStream in) throws IOException {
      Record template = inputRecord.copy();
      removeAttachments(template);
      MappingIterator iter = reader.readValues(in);
      try {
        while (iter.hasNextValue()) {
          Object rootNode = iter.nextValue();
          incrementNumRecords();
          LOG.trace("jsonObject: {}", rootNode);
         
          Record outputRecord = template.copy();
          outputRecord.put(Fields.ATTACHMENT_BODY, rootNode);
          outputRecord.put(Fields.ATTACHMENT_MIME_TYPE, MIME_TYPE);
 
          // pass record to next command in chain:
          if (!getChild().process(outputRecord)) {
            return false;
          }
View Full Code Here

    String path = RESOURCES_DIR;
    File sequenceFile = new File(path, "testSequenceFileContentSimple.seq");
    int numRecords = 5;
    HashMap<String, Record>  expected = createTextSequenceFile(sequenceFile, numRecords);
    InputStream in = new FileInputStream(sequenceFile.getAbsolutePath());
    Record record = new Record();
    record.put(Fields.ATTACHMENT_BODY, in);
    startSession();

    assertEquals(1, collector.getNumStartEvents());
    assertTrue(morphline.process(record));
    assertTrue(areFieldsEqual(expected, collector.getRecords()));
View Full Code Here

        SequenceFile.CompressionType.NONE, null, metadata);
      for (int i = 0; i < numRecords; ++i) {
        Text key = new Text("key" + i);
        Text value = new Text("value" + i);
        writer.append(key, value);
        Record record = new Record();
        record.put("key", key);
        record.put("value", value);
        map.put(key.toString(), record);
      }
    } finally {
      Closeables.closeQuietly(writer);
    }
View Full Code Here

    String path = RESOURCES_DIR;
    File sequenceFile = new File(path, "testSequenceFileContentCustomParsers.seq");
    int numRecords = 10;
    HashMap<String, Record> expected = createTextSequenceFile(sequenceFile, numRecords);
    InputStream in = new FileInputStream(sequenceFile.getAbsolutePath());
    Record record = new Record();
    record.put(Fields.ATTACHMENT_BODY, in);
    startSession();

    assertEquals(1, collector.getNumStartEvents());
    assertTrue(morphline.process(record));
   
View Full Code Here

        SequenceFile.CompressionType.NONE, null, metadata);
      for (int i = 0; i < numRecords; ++i) {
        Text key = new Text("key" + i);
        ParseTextMyWritableBuilder.MyWritable value = new ParseTextMyWritableBuilder.MyWritable("value", i);
        writer.append(key, value);
        Record record = new Record();
        record.put("key", key);
        record.put("value", value);
        map.put(key.toString(), record);
      }
    } finally {
      Closeables.closeQuietly(writer);
    }
View Full Code Here

    if (expected.size() != actual.size()) {
      return false;
    }
    for (Record current : actual) {
      String key = current.getFirstValue("key").toString();
      Record currentExpected = expected.get(key);
      if (!areRecordFieldsEqual(current, currentExpected, Arrays.asList(new String[]{"key", "value"}))) {
        return false;
      }
    }
View Full Code Here

TOP

Related Classes of org.kitesdk.morphline.api.Record

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.