Changeset 12902

Timestamp:

08/28/10 11:09:13 (13 years ago)

Author:

vvoutilainen

Message:

Fix reading of data containing scandinavian latin1 characters
correctly, and add a simple test for it. The utf-8 test is
just a sanity test so that umlauts as utf-8 aren't broken, the
latin1 test properly fails without this patch and passes
with this patch.

Location:

trunk/abcl

Files:

• abcl.asd (modified) (1 diff)
• src/org/armedbear/lisp/util/DecodingReader.java (modified) (4 diffs)
• src/org/armedbear/lisp/util/RandomAccessCharacterFile.java (modified) (1 diff)
• test/lisp/abcl/latin1-tests.lisp (added)
• test/lisp/abcl/latin1-umlauts.txt (added)
• test/lisp/abcl/utf8-umlauts.txt (added)

trunk/abcl/abcl.asd

@@ -46 +46 @@
                       (:file "math-tests")
                       (:file "misc-tests")
+                      (:file "latin1-tests")
                       (:file "bugs" :depends-on ("file-system-tests"))
                       (:file "pathname-tests")))))

trunk/abcl/src/org/armedbear/lisp/util/DecodingReader.java

@@ -46 +46 @@
 import java.nio.charset.CharsetEncoder;
 import java.nio.charset.CoderResult;
+import java.nio.charset.CodingErrorAction;
 
 import org.armedbear.lisp.Debug;
@@ -80 +81 @@
         this.stream = new PushbackInputStream(stream, size);
         this.cd = cs.newDecoder();
+        this.cd.onUnmappableCharacter(CodingErrorAction.REPLACE);
+        this.cd.onMalformedInput(CodingErrorAction.REPLACE);
         this.ce = cs.newEncoder();
         bbuf = ByteBuffer.allocate(size);
@@ -90 +93 @@
     public final void setCharset(Charset cs) {
         this.cd = cs.newDecoder();
+        this.cd.onUnmappableCharacter(CodingErrorAction.REPLACE);
+        this.cd.onMalformedInput(CodingErrorAction.REPLACE);
         this.ce = cs.newEncoder();
     }
@@ -258 +263 @@
 
         while (cb.remaining() > 0 && notEof) {
+            int oldRemaining = cb.remaining();
             notEof = ensureBbuf(forceRead);
             CoderResult r = cd.decode(bbuf, cb, ! notEof);
+            if (oldRemaining == cb.remaining()
+                && CoderResult.OVERFLOW == r) {
+                // if this happens, the decoding failed
+                // but the bufs didn't advance. Advance
+                // them manually and do manual replacing,
+                // otherwise we loop endlessly. This occurs
+                // at least when parsing latin1 files with
+                // lowercase o-umlauts in them.
+                // Note that this is at the moment copy-paste
+                // with RandomAccessCharacterFile.read()
+                cb.put('?');
+                bbuf.get();
+            }
             forceRead = (CoderResult.UNDERFLOW == r);
-
-            if (r.isMalformed()) {
-                throw new RACFMalformedInputException(bbuf.position(),
-                                                      (char)bbuf.get(bbuf.position()),
-                                                      cd.charset().name());
-            } else if (r.isUnmappable()) {
-                // a situation exactly like this is in DecodingReader too
-                Debug.assertTrue(false);
-            }
         }
         if (cb.remaining() == len)

trunk/abcl/src/org/armedbear/lisp/util/RandomAccessCharacterFile.java

@@ -371 +371 @@
         boolean atEof = false;
         while ((cbuf.remaining() > 0) && ! atEof) {
-
+            int oldRemaining = cbuf.remaining();
             atEof = ! ensureReadBbuf(decodeWasUnderflow);
             CoderResult r = cdec.decode(bbuf, cbuf, atEof );
+            if (oldRemaining == cbuf.remaining()
+                && CoderResult.OVERFLOW == r) {
+                // if this happens, the decoding failed
+                // but the bufs didn't advance. Advance
+                // them manually and do manual replacing,
+                // otherwise we loop endlessly. This occurs
+                // at least when parsing latin1 files with
+                // lowercase o-umlauts in them
+                // Note that this is at the moment copy-paste
+                // with DecodingReader.read()
+                cbuf.put('?');
+                bbuf.get();
+            }
             decodeWasUnderflow = (CoderResult.UNDERFLOW == r);
-            if (r.isMalformed())
-                // When reading encoded Unicode, we'd expect to require
-                // catching MalformedInput
-                throw new RACFMalformedInputException(bbuf.position(),
-                                                      (char)bbuf.get(bbuf.position()),
-                                                      cset.name());
-            if (r.isUnmappable())
-                // Since we're mapping TO unicode, we'd expect to be able
-                // to map all characters
-                Debug.assertTrue(false);
-            // OVERFLOW is a normal condition:
-            //  it's equal to cbuf.remaining() == 0
-            // ### EHU: really??? EXACTLY equal??
         }
         if (cbuf.remaining() == len) {