Examples of PlainTextByLineStream

Examples of opennlp.tools.util.PlainTextByLineStream

85
86
87
88
89
90
91
92
93
94
95
   *          the charset of the Arvores Deitadas Corpus
   */
  public ADChunkSampleStream(InputStream in, String charsetName) {

    try {
      this.adSentenceStream = new ADSentenceStream(new PlainTextByLineStream(
          in, charsetName));
    } catch (UnsupportedEncodingException e) {
      // UTF-8 is available on all JVMs, will never happen
      throw new IllegalStateException(e);
    }
View Full Code Here

Examples of opennlp.tools.util.PlainTextByLineStream

65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
      TokenNameFinderModel model = new TokenNameFinderModelLoader().load(new File(args[i]));
      nameFinders[i] = new NameFinderME(model);
    }
   
    ObjectStream<String> untokenizedLineStream =
        new PlainTextByLineStream(new InputStreamReader(System.in));
   
    PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "sent");
    perfMon.start();
   
    try {
      String line;
      while((line = untokenizedLineStream.read()) != null) {
        String whitespaceTokenizerLine[] = WhitespaceTokenizer.INSTANCE.tokenize(line);
       
        // A new line indicates a new document,
        // adaptive data must be cleared for a new document
       
View Full Code Here

Examples of opennlp.tools.util.PlainTextByLineStream

60
61
62
63
64
65
66
67
68
69
      File sampleDataFile, Charset encoding) {
    CmdLineUtil.checkInputFile(sampleDataName + " Data", sampleDataFile);

    FileInputStream sampleDataIn = CmdLineUtil.openInFile(sampleDataFile);

    ObjectStream<String> lineStream = new PlainTextByLineStream(sampleDataIn
        .getChannel(), encoding);

    return new TokenSampleStream(lineStream);
  }
View Full Code Here

Examples of opennlp.tools.util.PlainTextByLineStream

63
64
65
66
67
68
69
70
71
72
      File sampleDataFile, Charset encoding) {
    CmdLineUtil.checkInputFile(sampleDataName + " Data", sampleDataFile);

    FileInputStream sampleDataIn = CmdLineUtil.openInFile(sampleDataFile);

    ObjectStream<String> lineStream = new PlainTextByLineStream(sampleDataIn
        .getChannel(), encoding);

    return new NameSampleDataStream(lineStream);
  }
View Full Code Here

Examples of opennlp.tools.util.PlainTextByLineStream

96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
   
    Detokenizer detokenizer = new DictionaryDetokenizer(
        new DetokenizationDictionaryLoader().load(new File(args[0])));
   
    ObjectStream<String> tokenizedLineStream =
      new PlainTextByLineStream(new InputStreamReader(System.in));
   
    PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "sent");
    perfMon.start();
   
    try {
      String tokenizedLine;
      while ((tokenizedLine = tokenizedLineStream.read()) != null) {
       
        // white space tokenize line
        String tokens[] = WhitespaceTokenizer.INSTANCE.tokenize(tokenizedLine);
       
        DetokenizationOperation operations[] = detokenizer.detokenize(tokens);
View Full Code Here

Examples of opennlp.tools.util.PlainTextByLineStream

39
40
41
42
43
44
45
46
47
48
49
    InputStream in = getClass().getResourceAsStream(
        "/opennlp/tools/sentdetect/Sentences.txt");

    SentenceModel sentdetectModel = SentenceDetectorME.train(
        "en", new SentenceSampleStream(new PlainTextByLineStream(new InputStreamReader(in))), true, null, 100, 0);
   
    assertEquals("en", sentdetectModel.getLanguage());
   
    SentenceDetectorME sentDetect = new SentenceDetectorME(sentdetectModel);
View Full Code Here

Examples of opennlp.tools.util.PlainTextByLineStream

60
61
62
63
64
65
66
67
68
   
    InputStream trainDataIn = TokenizerTestUtil.class.getResourceAsStream(
        "/opennlp/tools/tokenize/token.train");
   
    ObjectStream<TokenSample> samples = new TokenSampleStream(
        new PlainTextByLineStream(new InputStreamReader(trainDataIn, "UTF-8")));
   
    return TokenizerME.train("en", samples, true, 5, 100);
  }
View Full Code Here

Examples of opennlp.tools.util.PlainTextByLineStream

226
227
228
229
230
231
232
233
234
235
236
        }
       
        additionalTrainingDataIn = new FileInputStream(additionalTrainingDataFile);
       
        ObjectStream<TokenSample> additionalSamples = new TokenSampleStream(
            new PlainTextByLineStream(new InputStreamReader(additionalTrainingDataIn, additionalTrainingDataEncoding)));
       
        samples = ObjectStreamUtils.createObjectStream(samples, additionalSamples);
      }
     
      tokenModel = TokenizerME.train(language, samples, isSkipAlphaNumerics);
View Full Code Here

Examples of opennlp.tools.util.PlainTextByLineStream

330
331
332
333
334
335
336
337
338
339
340
       
        additionalTrainingDataIn = new FileInputStream(additionalTrainingDataFile);
       
        // TODO: Make encoding configurable, otherwise use UTF-8 as default!
        ObjectStream<NameSample> additionalSamples = new NameSampleDataStream(
            new PlainTextByLineStream(new InputStreamReader(additionalTrainingDataIn, additionalTrainingDataEncoding)));
       
        samples = ObjectStreamUtils.createObjectStream(samples, additionalSamples);
      }
     
      // TODO: Make sure its possible to pass custom feature generator
View Full Code Here

Examples of opennlp.tools.util.PlainTextByLineStream

59
60
61
62
63
64
65
66
67
68
      File sampleDataFile, Charset encoding) {
    CmdLineUtil.checkInputFile(sampleDataName + " Data", sampleDataFile);

    FileInputStream sampleDataIn = CmdLineUtil.openInFile(sampleDataFile);

    ObjectStream<String> lineStream = new PlainTextByLineStream(sampleDataIn
        .getChannel(), encoding);

    return new ChunkSampleStream(lineStream);
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.