Package com.linkedin.databus2.schemas

Examples of com.linkedin.databus2.schemas.VersionedSchema

859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
        createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX,100, 200,j,newLine,0, -1, "", false, "");
        log.info("Directory is: " + dir);

        TrailFilePositionSetter posSetter = null;
        //GoldenGateTransactionSCNFinder finder = new GoldenGateTransactionSCNFinder();
        GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();

        //less than minScn
        log.info("less than MinScn case started !!");
        for (long i = 0 ; i < 100 ; i ++)
        {
          if ( true )break;
          posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
          //finder = new GoldenGateTransactionSCNFinder();
          finder = new GGXMLTrailTransactionFinder();

          FilePositionResult res = posSetter.getFilePosition(i,finder);
          Assert.assertEquals(res.getStatus(),FilePositionResult.Status.ERROR,"Result Status");
        }

        log.info("less than MinScn case passed !!");


        //Found Case
        for (long i = 100 ; i < 300 ; i ++)
        {
          posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
          finder = new GGXMLTrailTransactionFinder();

          log.info("SCN:  " + i);
          FilePositionResult res = posSetter.getFilePosition(i,finder);
          log.info("For scn (" + i + "):  the result is:  "  + res);
          if (i%2 == 0)
            assertFilePositionResult(res,dir,i+1,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
          else
            assertFilePositionResult(res,dir,i,FilePositionResult.Status.FOUND);

        }

        //Found Case
        FilePositionResult res = null;
        for (long i = 300 ; i < 320 ; i ++)
        {
          posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
          //finder = new GoldenGateTransactionSCNFinder();
          finder = new GGXMLTrailTransactionFinder();

          res = posSetter.getFilePosition(i,finder);
          //log.info("For scn (" + i + "):  the result is:  "  + res);
          assertFilePositionResult(res,dir,299,FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
        }

        // USE Latest SCN (-1)
        posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
        //finder = new GoldenGateTransactionSCNFinder();
        finder = new GGXMLTrailTransactionFinder();

        res = posSetter.getFilePosition(-1,finder);
        //log.info("For scn (" + i + "):  the result is:  "  + res);
        assertFilePositionResult(res,dir,299,FilePositionResult.Status.FOUND);
      }
View Full Code Here
169
170
171
172
173
174
175
176
177
178
179
    if(uri == null)
      throw new DatabusException("Uri is required to start the relay");
    uri = uri.trim();
    EventProducer producer = null;
    if (uri.startsWith("jdbc:")) {
      SourceType sourceType = pConfig.getReplBitSetter().getSourceType();
          if (SourceType.TOKEN.equals(sourceType))
            throw new DatabusException("Token Source-type for Replication bit setter config cannot be set for trigger-based Databus relay !!");

      // if a buffer for this partiton exists - we are overwri
      producer = new OracleEventProducerFactory().buildEventProducer(
View Full Code Here
129
130
131
132
133
134
135
136
137
          break;

        default:
          break;
      }
      SchemaId schemaId = schemaStr != null ? SchemaId.createWithMd5(schemaStr) : null;
      byte[] schemaBytes = schemaId != null? schemaId.getByteArray(): new byte[16];
      initBufferWithEvents(eventsBuf, keyBase, numEvents, srcId,schemaBytes, keyCounts, srcidCounts);
    }
View Full Code Here
147
148
149
150
151
152
153
154
155
156
157
    // Get the event buffer
    DbusEventBufferAppendable dbusEventBuffer = getEventBuffer()
        .getDbusEventBufferAppendable(pPartition);

    // Get the schema registry service
    SchemaRegistryService schemaRegistryService = getSchemaRegistryService();

    // Get a stats collector per physical source
    addPhysicalPartitionCollectors(pPartition);
    String statsCollectorName = pPartition.toSimpleString();
    /*
 
View Full Code Here
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
                     srcidCounts.get((short)2).intValue());

        //check metadata schemas

        EventCountingConsumer myCons = (EventCountingConsumer) mockConsumer;
        VersionedSchema metadataSchema = myCons.getMetadataSchema();
        Assert.assertTrue(null != metadataSchema);

        log.info("Metadata VersionedSchema = " + metadataSchema);
        Assert.assertEquals(metadataSchema.getVersion(),2);
        Assert.assertEquals(metadataSchema.getSchemaBaseName(), SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE);

        verifyNoLocks(log, eventsBuf);
        log.info("end\n");
    }
View Full Code Here
263
264
265
266
267
268
269
270
271
272
273
274
275
276
    dbName = dbName.trim().toLowerCase();
    config.setName(dbName);
    config.setUri(uri);
    for (String srcName : srcNames)
    {
      VersionedSchema schema = null;
      schema = s.fetchLatestVersionedSchemaBySourceName(srcName);

      String dbObjectName = SchemaHelper.getMetaField(schema.getSchema(), "dbFieldName");

      LogicalSourceConfig c = new LogicalSourceConfig();
      c.setId(manager.getSrcId(srcName));
      c.setName(srcName);
      c.setUri(dbName + "." + dbObjectName);
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
    _staticConfigBuilder.setSourceName("5002", "test5.source2");

    _schemaRegistry = new VersionedSchemaSetBackedRegistryService();

    String schema31Str = "{\"name\":\"source1_v1\",\"namespace\":\"test3\",\"type\":\"record\",\"fields\":[{\"type\":\"int\",\"name\":\"intField\"}]}";
    VersionedSchema vschema31 = new VersionedSchema("test3.source1", (short)1, Schema.parse(schema31Str), null);
    String schema32Str = "{\"name\":\"source2_v1\",\"namespace\":\"test3\",\"type\":\"record\",\"fields\":[{\"type\":\"string\",\"name\":\"strField\"}]}";
    VersionedSchema vschema32 = new VersionedSchema("test3.source2", (short)1, Schema.parse(schema32Str), null);
    _schemaRegistry.registerSchema(vschema31);
    _schemaRegistry.registerSchema(vschema32);

    String schema41Str = "{\"name\":\"source1_v1\",\"namespace\":\"test4\",\"type\":\"record\",\"fields\":[{\"type\":\"int\",\"name\":\"intField\"}]}";
    VersionedSchema vschema41 = new VersionedSchema("test4.source1", (short)1, Schema.parse(schema41Str), null);
    String schema42Str = "{\"name\":\"source2_v1\",\"namespace\":\"test4\",\"type\":\"record\",\"fields\":[{\"type\":\"string\",\"name\":\"strField\"}]}";
    VersionedSchema vschema42 = new VersionedSchema("test4.source2", (short)1, Schema.parse(schema42Str), null);
    _schemaRegistry.registerSchema(vschema41);
    _schemaRegistry.registerSchema(vschema42);

    String schema51Str = "{\"name\":\"source1_v1\",\"namespace\":\"test5\",\"type\":\"record\",\"fields\":[{\"type\":\"int\",\"name\":\"intField\"}]}";
    VersionedSchema vschema51 = new VersionedSchema("test5.source1", (short)1, Schema.parse(schema51Str), null);
    //String schema52Str = "{\"name\":\"source2_v1\",\"namespace\":\"test5\",\"type\":\"record\",\"fields\":[{\"type\":\"string\",\"name\":\"strField\"}]}";
    VersionedSchema vschema52 = new VersionedSchema("test5.source2", (short)1, Schema.parse(schema51Str), null);
    _schemaRegistry.registerSchema(vschema51);
    _schemaRegistry.registerSchema(vschema52);

    // See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Espresso+Metadata+Schema for
    // the real definition.
    String metadataSchemaStr = "{\"name\":\"metadata\",\"namespace\":\"test_namespace\",\"type\":\"record\",\"fields\":[{\"name\":\"randomStrField\",\"type\":\"string\"}]}";
    // TODO (DDSDBUS-2093/2096):  update this according to real schema-registry metadata API.
    // (Right now the "metadata" string here is used as a key for lookups within the relay's
    // schema registry, but that could conflict with an actual source schema named "metadata".)
    VersionedSchema metaSchema = new VersionedSchema("metadata", (short)1, Schema.parse(metadataSchemaStr), null);
    _schemaRegistry.registerSchema(metaSchema);

    _staticConfigBuilder.getSchemaRegistry().setType(RegistryType.EXISTING.toString());
    _staticConfigBuilder.getSchemaRegistry().useExistingService(_schemaRegistry);
    _staticConfig = _staticConfigBuilder.build();
View Full Code Here
215
216
217
218
219
220
221
222
223
224
225
    SchemaRegistryService schemaRegistry =
        FileSystemSchemaRegistryService.build(_bsStaticConfig.getSchemaRegistry().getFileSystem());
    LOG.info("Schema = " + schemaRegistry.fetchLatestSchemaBySourceName(_queryStaticConfig.getSourceName()));

    _schema = Schema.parse(schemaRegistry.fetchLatestSchemaBySourceName(_queryStaticConfig.getSourceName()));
    VersionedSchema vs = new VersionedSchema(_schema.getFullName(), (short)1, _schema, null);

    VersionedSchemaSet schemaSet = new VersionedSchemaSet();
    schemaSet.add(vs);
    _decoder = new DbusEventAvroDecoder(schemaSet);
  }
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
        Iterator<Map.Entry<Short, String>> it = schemaSet.entrySet().iterator();
        while (it.hasNext())
        {
          Map.Entry<Short, String> pairs = it.next();
          Schema s = Schema.parse(pairs.getValue());
          VersionedSchema vs = new VersionedSchema(s.getFullName(),
              pairs.getKey(), s, null);
          vSchemaSet.add(vs);
        }

        /* Try and identify the schema key */
        VersionedSchema vschema = schemaRegistry
            .fetchLatestVersionedSchemaBySourceName(source.getSourceName());
        Schema schema = Schema.parse(vschema.getSchema().toString());
        LOG.info("Schema =" + vschema.getSchema() + "version="
            + vschema.getVersion() + " name=" + vschema.getSchemaBaseName());

        /* Determine type of field txn */
        Field txnFieldType = schema.getField("txn");
        if (txnFieldType == null)
        {
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
  public GenericRecord getGenericRecord(DbusEvent e, GenericRecord reuse)
  {
    byte[] md5 = new byte[16];
    e.schemaId(md5);
    SchemaId schemaId = new SchemaId(md5);
    VersionedSchema writerSchema = _schemaSet.getById(schemaId);

    if (null == writerSchema)
    {
      LOG.error("Unable to find schema for id " + schemaId + "; event = " + e);
      throw new DatabusRuntimeException("No schema available to decode event " + e);
    }

    ByteBuffer valueBuffer = e.value();

    byte[] valueBytes = null;
    if (valueBuffer.hasArray())
    {
      valueBytes = valueBuffer.array();
    }
    else
    {
      valueBytes = new byte[valueBuffer.remaining()];
      valueBuffer.get(valueBytes);
    }

    return getGenericRecord(valueBytes, writerSchema.getSchema(), reuse);
  }
View Full Code Here
TOP

Related Classes of com.linkedin.databus2.schemas.VersionedSchema

  • com.google.code.or.common.glossary.Row
  • com.linkedin.databus.bootstrap.server.BootstrapHttpServer
  • com.linkedin.databus.bootstrap.utils.BootstrapAvroFileEventReader
  • com.linkedin.databus.bootstrap.utils.BootstrapAvroFileSeederMain
  • com.linkedin.databus.bootstrap.utils.BootstrapTableReader
  • com.linkedin.databus.client.BootstrapPullThread
  • com.linkedin.databus.client.consumer.TestInternalMetadata
  • com.linkedin.databus.client.DatabusHttpClientImpl
  • com.linkedin.databus.client.DatabusSourcesConnection
  • com.linkedin.databus.client.DatabusSourcesConnection$Config
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.