Package org.kitesdk.morphline.api

Examples of org.kitesdk.morphline.api.Record


    createRCFile(fileName, numRecords, maxColumns);
    List<Record> expected = Lists.newArrayList();
    if (rowWise) {
      // Row wise expected records
      for (int row = 0; row < numRecords; row++) {
        Record record = new Record();
        for (int column = 0; column < maxColumns; column++) {
          Text sampleText = new Text("ROW-NUM:" + row + ", COLUMN-NUM:"
              + column);
          record.put("field" + (column + 1), sampleText);
        }
        expected.add(record);
      }
    } else {
      // Column wise expected records
      for (int column = 0; column < maxColumns; column++) {
        Record record = new Record();
        List<Writable> outputs = new ArrayList<Writable>();
        for (int row = 0; row < numRecords; row++) {
          Text sampleText = new Text("ROW-NUM:" + row + ", COLUMN-NUM:"
              + column);
          outputs.add(sampleText);
        }
        record.put("field" + (column + 1), outputs);
        expected.add(record);
      }
    }
    return expected;
  }
View Full Code Here


      return false;
    }

    if (rowWiseCheck) {
      for (int i = 0; i < actual.size(); i++) {
        Record currentExpected = expected.get(i);
        Record currentActual = actual.get(i);
        if (!areRecordColumnsEqual(currentActual, currentExpected, columnSize)) {
          return false;
        }
      }
    } else {
      for (int i = 0; i < columnSize; i++) {
        String fieldName = "field" + (i + 1);
        Record currentExpected = expected.get(i);
        Record currentActual = actual.get(i);
        if (!isRecordColumnEqual(currentActual, currentExpected, fieldName)) {
          return false;
        }
      }
    }
View Full Code Here

  @Test
  public void testLoadSolrBasic() throws Exception {
    //System.setProperty("ENV_SOLR_HOME", testSolrHome + File.separator + "collection1");
    morphline = createMorphline("test-morphlines" + File.separator + "loadSolrBasic");   
    //System.clearProperty("ENV_SOLR_HOME");
    Record record = new Record();
    record.put(Fields.ID, "id0");
    record.put("first_name", "Nadja"); // will be sanitized
    startSession();
    Notifications.notifyBeginTransaction(morphline);
    assertTrue(morphline.process(record));
    assertEquals(1, collector.getNumStartEvents());
    Notifications.notifyCommitTransaction(morphline);
    Record expected = new Record();
    expected.put(Fields.ID, "id0");
    assertEquals(Arrays.asList(expected), collector.getRecords());
    assertEquals(1, queryResultSetSize("*:*"));
    Notifications.notifyRollbackTransaction(morphline);
    Notifications.notifyShutdown(morphline);
  }
View Full Code Here

 
  public void testLoadSolrWithPartialUpdate() throws Exception {
    morphline = createMorphline("test-morphlines" + File.separator + "loadSolrBasic");   
   
    // insert a document with a bunch of fields
    Record record = new Record();
    record.put(Fields.ID, "id0");
    record.put("user_friends_count", 123);
    record.put("text", "myText");
    Notifications.notifyBeginTransaction(morphline);
    assertTrue(morphline.process(record));
    assertEquals(1, collector.getRecords().size());
    assertEquals(1, query("*:*").getResults().size());

    // remove "text" field; retain other fields as-is
    record = new Record();
    record.put(Fields.ID, "id0");
    Map<String, Object> map = new HashMap();
    map.put("set", null);
    record.put("text", map);
    assertTrue(morphline.process(record));   
    SolrDocumentList docs = query("*:*").getResults();
    assertEquals(1, docs.size());
    assertEquals("id0", docs.get(0).getFirstValue(Fields.ID));
    assertEquals(123, docs.get(0).getFirstValue("user_friends_count"));
    assertNull(docs.get(0).getFieldValue("text"));

    // set "text" field to "hello world"; retain other fields as-is
    record = new Record();
    record.put(Fields.ID, "id0");
    record.put("text", Collections.singletonMap("set", "hello world"));
    assertTrue(morphline.process(record));   
    docs = query("*:*").getResults();
    assertEquals(1, docs.size());
    assertEquals("id0", docs.get(0).getFirstValue(Fields.ID));
    assertEquals(123, docs.get(0).getFirstValue("user_friends_count"));
    assertEquals("hello world", docs.get(0).getFirstValue("text"));
   
    // add "goodbye moon" to text field; retain other fields as-is
    record = new Record();
    record.put(Fields.ID, "id0");
    record.put("text", ImmutableMap.of("add", "goodbye moon"));
    assertTrue(morphline.process(record));   
    docs = query("*:*").getResults();
    assertEquals(1, docs.size());
    assertEquals("id0", docs.get(0).getFirstValue(Fields.ID));
    assertEquals(123, docs.get(0).getFirstValue("user_friends_count"));
    assertEquals(Arrays.asList("hello world", "goodbye moon"), docs.get(0).get("text"));   

    // set "text" field to multiple values ["hello sun", "goodbye mars"]; retain other fields as-is
    record = new Record();
    record.put(Fields.ID, "id0");
    record.put("text", ImmutableMap.of("set", Arrays.asList("hello sun", "goodbye mars")));
    assertTrue(morphline.process(record));   
    docs = query("*:*").getResults();
    assertEquals(1, docs.size());
    assertEquals("id0", docs.get(0).getFirstValue(Fields.ID));
    assertEquals(123, docs.get(0).getFirstValue("user_friends_count"));
    assertEquals(Arrays.asList("hello sun", "goodbye mars"), docs.get(0).getFieldValue("text"));
   
    // increment user_friends_count by 5; retain other fields as-is
    record = new Record();
    record.put(Fields.ID, "id0");
    record.put("user_friends_count", ImmutableMap.of("inc", 5));
    assertTrue(morphline.process(record));   
    docs = query("*:*").getResults();
    assertEquals(1, docs.size());
    assertEquals("id0", docs.get(0).getFirstValue(Fields.ID));
    assertEquals(128, docs.get(0).getFirstValue("user_friends_count"));
View Full Code Here

      Path attachmentPath = getAttachmentPath(record);
      SingleStreamFileSystem fs = new SingleStreamFileSystem(in, attachmentPath);
      RCFile.Reader reader = null;
      try {
        reader = new RCFile.Reader(fs, attachmentPath, conf);
        Record template = record.copy();
        removeAttachments(template);
        template.put(Fields.ATTACHMENT_MIME_TYPE, OUTPUT_MEDIA_TYPE);
        if (includeMetaData) {
          SequenceFile.Metadata metadata = reader.getMetadata();
          if (metadata != null) {
            template.put(RC_FILE_META_DATA, metadata);
          }
        }
       
        switch (readMode) {
        case row:
View Full Code Here

        if (!next) {
          break;
        }

        incrementNumRecords();
        Record outputRecord = record.copy();       
        BytesRefArrayWritable rowBatchBytes = new BytesRefArrayWritable();
        rowBatchBytes.resetValid(columns.size());
        reader.getCurrentRow(rowBatchBytes);

        // Read all the columns configured and set it in the output record
        for (RCFileColumn rcColumn : columns) {
          BytesRefWritable columnBytes = rowBatchBytes.get(rcColumn.getInputField());
          outputRecord.put(rcColumn.getOutputField(), updateColumnValue(rcColumn, columnBytes));
        }
       
        // pass record to next command in chain:
        if (!getChild().process(outputRecord)) {
          return false;
View Full Code Here

        for (RCFileColumn rcColumn : columns) {
          BytesRefArrayWritable rowBatchBytes = reader.getColumn(rcColumn.getInputField(), null);
          List<Writable> rowValues = new ArrayList<Writable>(rowBatchBytes.size());
          incrementNumRecords();
          Record outputRecord = record.copy();
          for (int rowIndex = 0; rowIndex < rowBatchBytes.size(); rowIndex++) {
            BytesRefWritable rowBytes = rowBatchBytes.get(rowIndex);
            rowValues.add(updateColumnValue(rcColumn, rowBytes));
          }
          outputRecord.put(rcColumn.getOutputField(), rowValues);
         
          // pass record to next command in chain:
          if (!getChild().process(outputRecord)) {
            return false;
          }
View Full Code Here

  @Test
  public void testLoadSolrWithDelete() throws Exception {
    morphline = createMorphline("test-morphlines" + File.separator + "loadSolrBasic");   

    // insert
    Record record = new Record();
    record.replaceValues(Fields.ID, "id0");
    record.replaceValues("first_name", "Nadja"); // will be sanitized
    Notifications.notifyBeginTransaction(morphline);
    assertTrue(morphline.process(record.copy()));
    assertEquals(1, query("*:*").getResults().size());
   
    // insert
    record = new Record();
    record.replaceValues(Fields.ID, "id1");
    assertTrue(morphline.process(record.copy()));
    assertEquals(2, query("*:*").getResults().size());
   
    // deleteById
    record = new Record();
    record.replaceValues(LoadSolrBuilder.LOAD_SOLR_DELETE_BY_ID, "id0");
    assertTrue(morphline.process(record.copy()));
    assertEquals(1, query("*:*").getResults().size());
   
    // deleteById
    record = new Record();
    record.replaceValues(LoadSolrBuilder.LOAD_SOLR_DELETE_BY_ID, "idNonExistent");
    assertTrue(morphline.process(record.copy()));
    assertEquals(1, query("*:*").getResults().size());

    // insert
    record = new Record();
    record.replaceValues(Fields.ID, "id2");
    assertTrue(morphline.process(record.copy()));
    assertEquals(2, query("*:*").getResults().size());
   
    // insert
    record = new Record();
    record.replaceValues(Fields.ID, "id200");
    assertTrue(morphline.process(record.copy()));
    assertEquals(3, query("*:*").getResults().size());

    // deleteByQuery
    record = new Record();
    record.replaceValues(LoadSolrBuilder.LOAD_SOLR_DELETE_BY_QUERY, "id:id2*");
    record.put(LoadSolrBuilder.LOAD_SOLR_DELETE_BY_QUERY, "text:NonExistent");
    assertTrue(morphline.process(record.copy()));
    assertEquals(1, query("*:*").getResults().size());
   
    // deleteByQuery
    record = new Record();
    record.put(LoadSolrBuilder.LOAD_SOLR_DELETE_BY_QUERY, "id:NonExistent");
    record.put(LoadSolrBuilder.LOAD_SOLR_DELETE_BY_QUERY, "text:NonExistent");
    assertTrue(morphline.process(record.copy()));
    assertEquals(1, query("*:*").getResults().size());
   
    // insert
    record = new Record();
    record.replaceValues(Fields.ID, "id3");
    assertTrue(morphline.process(record.copy()));
    assertEquals(2, query("*:*").getResults().size());

    // verify
    SolrDocumentList docs = query("*:*").getResults();
    assertEquals(2, docs.size());
View Full Code Here

   
  @Test
  public void testTokenizeText() throws Exception {
    morphline = createMorphline("test-morphlines" + File.separator + "tokenizeText");
    for (int i = 0; i < 3; i++) {
      Record record = new Record();
      record.put(Fields.MESSAGE, "Hello World!");
      record.put(Fields.MESSAGE, "\nFoo@Bar.com #%()123");
      Record expected = record.copy();
      expected.getFields().putAll("tokens", Arrays.asList("hello", "world", "foo", "bar.com", "123"));
      collector.reset();
      startSession();
      Notifications.notifyBeginTransaction(morphline);
      assertTrue(morphline.process(record));
      assertEquals(1, collector.getNumStartEvents());
View Full Code Here

  public void testReadProto() throws Exception {
    morphline = createMorphline("test-morphlines/readProto");
    for (int j = 0; j < 3; j++) { // also test reuse of objects and low level
                                  // avro buffers
      InputStream in = new ByteArrayInputStream(repeatedLongs.toByteArray());
      Record record = new Record();
      record.put(Fields.ATTACHMENT_BODY, in);

      collector.reset();
      startSession();
      assertEquals(1, collector.getNumStartEvents());
      assertTrue(morphline.process(record));
View Full Code Here

TOP

Related Classes of org.kitesdk.morphline.api.Record

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.