Package org.mozilla.universalchardet

Examples of org.mozilla.universalchardet.UniversalDetector

44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
    public String getDetectedEncoding(File file) {
        InputStream is = null;
        String encoding = null;
        try {
            is = new FileInputStream(file);
            UniversalDetector detector = new UniversalDetector(null);
            byte[] buf = new byte[4096];
            int nread;
            while ((nread = is.read(buf)) > 0 && !detector.isDone()) {
                detector.handleData(buf, 0, nread);
            }
            detector.dataEnd();
            encoding = detector.getDetectedCharset();
        } catch (IOException e) {
            // nothing to do
        } finally {
            IOUtil.close(is);
            if (encoding == null) {
View Full Code Here
11
12
13
14
15
16
17
18
19
20
21
22
23
24
    private UniversalDetectorUtil() {
    }

    public static String getDetectedEncoding(InputStream is) throws IOException {
        UniversalDetector detector = new UniversalDetector(null);
        byte[] buf = new byte[4096];
        int nread;
        while ((nread = is.read(buf)) > 0 && !detector.isDone()) {
            detector.handleData(buf, 0, nread);
        }
        detector.dataEnd();
        return detector.getDetectedCharset();
    }
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
  public String getDetectedEncoding(File file) {
    InputStream is = null;
    String encoding = null;
    try {
      is = new FileInputStream(file);
      UniversalDetector detector = new UniversalDetector(null);
      byte[] buf = new byte[4096];
      int nread;
      while ((nread = is.read(buf)) > 0 && !detector.isDone()) {
        detector.handleData(buf, 0, nread);
      }
      detector.dataEnd();
      encoding = detector.getDetectedCharset();
    } catch (IOException e) {
      // nothing to do
    } finally {
      IOUtil.close(is);
      if (encoding == null) {
View Full Code Here
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
                in = new FileInputStream(file);
                inStream = new DataInputStream(in);
      byte[] buf = new byte[(int)file.length()];
      int nrRead = inStream.read(buf);
   
      UniversalDetector detector = new UniversalDetector(null);
      detector.handleData(buf, 0, nrRead);
      detector.dataEnd();

      String encoding = detector.getDetectedCharset();
      detector.reset();
      if (encoding != null) {
        if (!encoding.equals(ENCODING)) {
          Log.error(Log.JEEVES,"Detected character set "+encoding+", converting to UTF-8");
          return convertByteArrayToUTF8ByteArray(buf, encoding);
        }
View Full Code Here
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
   *         or null if not detected
   * @throws IOException
   */
  public static String getFileCharset(File file) throws IOException {
    byte[] buf = new byte[4096];
    final UniversalDetector universalDetector;
    try (BufferedInputStream bufferedInputStream = new BufferedInputStream(new FileInputStream(file))) {
      universalDetector = new UniversalDetector(null);
      int numberOfBytesRead;
      while ((numberOfBytesRead = bufferedInputStream.read(buf)) > 0 && !universalDetector.isDone()) {
        universalDetector.handleData(buf, 0, numberOfBytesRead);
      }
    }
    universalDetector.dataEnd();
    String encoding = universalDetector.getDetectedCharset();

    if (encoding != null) {
      LOGGER.debug("Detected encoding for {} is {}.", file.getAbsolutePath(), encoding);
    } else {
      LOGGER.debug("No encoding detected for {}.", file.getAbsolutePath());
    }

    universalDetector.reset();

    return encoding;
  }
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
        return content;
    }

    private static String detectEncoding(byte[] content)
    {
        UniversalDetector detector = new UniversalDetector(null);
        detector.handleData(content, 0, content.length);
        detector.dataEnd();
        String encoding = detector.getDetectedCharset();
        if (encoding != null)
        {
            LOGGER.debug(String.format("Detected encoding: %s\n", encoding));
        }
        else
        {
            encoding = getDefaultEncoding();
            LOGGER.debug(String.format("No encoding detected, using default: %s\n", encoding));
        }
        detector.reset();
        return encoding;
    }
View Full Code Here
475
476
477
478
479
480
481
482
483
484
485
486
487
488
   * @throws Exception
   */
  private void checkForBadCharacters(String json) throws Exception
  {
    byte[] bytes = json.getBytes();   
    UniversalDetector ud = new UniversalDetector(null);
    ud.handleData(bytes, 0, bytes.length);
    ud.dataEnd();
    String encoding = ud.getDetectedCharset();     
    if ( encoding != null )
    {
      //do an extra check for charcode 65533 if encoding is utf-8   
      if ( encoding.equals("UTF-8"))
      {
View Full Code Here
407
408
409
410
411
412
413
414
415
416
417
418
419
420
   * @throws Exception
   */
  private void checkForBadCharacters(String json) throws Exception
  {
    byte[] bytes = json.getBytes();   
    UniversalDetector ud = new UniversalDetector(null);
    ud.handleData(bytes, 0, bytes.length);
    ud.dataEnd();
    String encoding = ud.getDetectedCharset();     
    if ( encoding != null )
    {
      //do an extra check for charcode 65533 if encoding is utf-8   
      if ( encoding.equals("UTF-8"))
      {
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
     * @param bytes - a byte array to be checked
     * @return charset - the charset used
     * @throws IOException
     */
    public static String detectCharset(byte bytes[]) {
        UniversalDetector detector = new UniversalDetector(null);
        int offset = 0;

        do {
            int blockSize = Math.min(4096, bytes.length - offset);
            detector.handleData(bytes, offset, blockSize);
            offset += blockSize;
        } while(offset < bytes.length);
        detector.dataEnd();

        return or(detector.getDetectedCharset(), "UTF-8");
    }
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
     * @param is - an input stream to be checked
     * @return charset - the charset used
     * @throws IOException
     */
    public static String detectCharset(InputStream is) throws IOException {
        UniversalDetector detector = new UniversalDetector(null);
        byte[] buf = new byte[4096];
        int nRead;

        while ((nRead = is.read(buf)) > 0 && !detector.isDone()) {
            detector.handleData(buf, 0, nRead);
        }
        detector.dataEnd();

        return or(detector.getDetectedCharset(), "UTF-8");
    }
View Full Code Here
TOP

Related Classes of org.mozilla.universalchardet.UniversalDetector

  • cn.edu.hfut.dmic.webcollector.util.CharsetDetector
  • com.commafeed.backend.feed.FeedUtils
  • com.google.code.ftspc.lector.indexers.CommonFunctions
  • com.google.gerrit.server.patch.Text
  • com.ikanow.infinit.e.api.social.sharing.ShareInterface
  • com.ikanow.infinit.e.api.social.sharing.ShareV2Interface
  • com.vtence.molecule.support.CharsetDetector
  • de.jungblut.crawl.extraction.OutlinkExtractor
  • edu.isi.karma.util.EncodingDetector
  • net.pms.util.FileUtil
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.