Package com.ibm.icu.text

Examples of com.ibm.icu.text.CharsetDetector

100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
      }

      // charset detection with icu
      try {
        bis = new BufferedInputStream(new FileInputStream(tempFile));
        CharsetDetector detector;
        detector = new CharsetDetector();
        detector.enableInputFilter(true);
        detector.setText(bis);
        if (declaredEncoding!=null && !"".equals(declaredEncoding))
          detector.setDeclaredEncoding(declaredEncoding);
        CharsetMatch[] matches = null;
        matches = detector.detectAll();
        bis.close();
        encoding = HttpUtils.filtreEncoding(matches[0].getName().toLowerCase());
        if (encoding!=null && !"".equals(encoding))
        {
          if (encodingFreq.containsKey(encoding))
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
          encodingFreq.put(encoding, 2);
      }

      // charset detection with icu
      bis = new BufferedInputStream(new FileInputStream(tempFile));
      CharsetDetector detector;
      detector = new CharsetDetector();
      detector.enableInputFilter(true);
      detector.setText(bis);
      if (declaredEncoding!=null && !"".equals(declaredEncoding))
        detector.setDeclaredEncoding(declaredEncoding);
      CharsetMatch[] matches = detector.detectAll();
      bis.close();
      encoding = HttpUtils.filtreEncoding(matches[0].getName().toLowerCase());
      if (encoding!=null && !"".equals(encoding)) {
        if (encodingFreq.containsKey(encoding))
          encodingFreq.put(encoding, encodingFreq.get(encoding) + 2);
View Full Code Here
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
    }

    @Override
    protected CoderResult implFlush(CharBuffer out) {
      if (usedDecoder == null) {
        CharsetDetector detector = new CharsetDetector();
        detector.enableInputFilter(filtered);
        byte[] data = buffer.toByteArray();
        detector.setText(data);
        CharsetMatch cm = detector.detect();
        try {
          usedDecoder = Charset.forName(cm == null ? "ISO-8859-1" : cm.getName()).newDecoder();
        } catch (UnsupportedCharsetException ex) {
          usedDecoder = Charset.forName("ISO-8859-1").newDecoder();
        }
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
66
    public static Reader readerWithCharsetDetect(File f) throws FileNotFoundException {
        return readerWithCharsetDetect(new BufferedInputStream(new FileInputStream(f)));
    }

    public static Reader readerWithCharsetDetect(InputStream is) {
        CharsetDetector detector = new CharsetDetector();
        try {
            CharsetMatch match = detector.setText(is).detect();
            is.reset();
            return new InputStreamReader(is, match.getName());
        } catch (IOException e) {
            e.printStackTrace();
            try {
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
58
59
        return source.readByte();
    }
   
    public Encoding sniff() throws IOException {
        try {
            CharsetDetector detector = new CharsetDetector();
            detector.setText(this);
            CharsetMatch match = detector.detect();
            Encoding enc = Encoding.forName(match.getName());
            Encoding actual = enc.getActualHtmlEncoding();
            if (actual != null) {
                enc = actual;
            }
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
66
  public static BufferedReader getSafeReaderFor(String name) throws FileNotFoundException {
    InputStream stream = getStreamFor(name);
    if( stream!=null ) {   
      Charset charset;
      try {
        CharsetDetector detector = new CharsetDetector();
        detector.setText(new BufferedInputStream(stream));
        charset = Charset.forName( detector.detect().getName() );
      } catch (Exception e) {
        charset = Charset.defaultCharset();
      }
      return new BufferedReader(new InputStreamReader(getStreamFor(name), charset));
    } else {
View Full Code Here
TOP

Related Classes of com.ibm.icu.text.CharsetDetector

  • com.ibm.icu.dev.demo.charsetdet.DetectingViewer
  • com.ibm.icu.dev.test.charsetdet.TestCharsetDetector
  • com.nardoz.restopengov.utils.CSVFetcher
  • fr.eolya.utils.http.HttpStream
  • fr.eolya.utils.http.WebStream
  • net.geco.basics.GecoResources
  • net.sf.jmatchparser.util.charset.icu4jchardet.ICU4JChardetCharset$Decoder
  • net.vidageek.crawler.component.WebDownloader
  • net.yacy.document.parser.htmlParser
  • nu.validator.htmlparser.extra.IcuDetectorSniffer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.