1 ;;; utf-16.el --- UTF-16 encoding/decoding
3 ;; Copyright (C) 2001, 2002, 2003, 2004 Free Software Foundation, Inc.
5 ;; Author: Dave Love <fx@gnu.org>
6 ;; Keywords: Unicode, UTF-16, i18n
8 ;; This file is part of GNU Emacs.
10 ;; GNU Emacs is free software; you can redistribute it and/or modify
11 ;; it under the terms of the GNU General Public License as published by
12 ;; the Free Software Foundation; either version 2, or (at your option)
15 ;; GNU Emacs is distributed in the hope that it will be useful,
16 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
17 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 ;; GNU General Public License for more details.
20 ;; You should have received a copy of the GNU General Public License
21 ;; along with GNU Emacs; see the file COPYING. If not, write to the
22 ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
23 ;; Boston, MA 02111-1307, USA.
27 ;; Support for UTF-16, which is a two-byte encoding (modulo
28 ;; surrogates) of Unicode, defined in RFC 2781. It is written either
29 ;; in little or big endian order and either with or without the
30 ;; leading BOM (a two-byte signature which identifies their byte sex).
32 ;; We provide these base coding systems.
35 ;; mule-utf-16le little no
36 ;; mule-utf-16be big no
37 ;; mule-utf-16le-with-signature little yes
38 ;; mule-utf-16be-with-signature big yes
39 ;; mule-utf-16 both yes
41 ;; Note that un-decodable sequences aren't (yet?) preserved as raw
42 ;; bytes, as they are with utf-8, so reading and writing as utf-16 can
47 ;; We end up with trivially different -le and -be versions of most
48 ;; things below, sometimes with commonality abstracted into a let
49 ;; binding for maintenance convenience.
51 ;; Needed in macro expansion, so can't be let-bound. Zapped after use.
53 (defconst utf-16-decode-ucs
54 ;; If r5 is negative, r1 is a Unicode chacter code. Otherise, r5 is
55 ;; the first of a surrogate pair and r1 is the second of the pair.
56 ;; Output is charset ID in r0, code point in r1. R0 may be set to
57 ;; -1 in which case a caller should not write out r1.
60 (if ((r1 >= #xE000
) | r0
)
61 ;; Invalid second code of surrogate pair.
63 (call ccl-mule-utf-untrans
))
65 (r1 += (((r5 -
#xD800
) << 10) + #x10000
))))
68 (r0 = ,(charset-id 'ascii
))
69 ((lookup-integer utf-subst-table-for-decode r1 r3
)
70 (if r7
; got a translation
73 (r0 = ,(charset-id 'eight-bit-control
))
75 ((r0 = ,(charset-id 'latin-iso8859-1
))
78 ((r0 = ,(charset-id 'mule-unicode-0100-24ff
))
80 (r2 = (((r1 / 96) + 32) << 7))
84 ((r0 = ,(charset-id 'mule-unicode-2500-33ff
))
86 (r2 = (((r1 / 96) + 32) << 7))
90 ;; We can't have this character.
92 (call ccl-mule-utf-untrans
)
96 ;; The first code of a surrogate pair.
100 ;; The second code of a surrogate pair, invalid.
102 (call ccl-mule-utf-untrans
)
106 ((r0 = ,(charset-id 'mule-unicode-e000-ffff
))
108 (r2 = (((r1 / 96) + 32) << 7))
111 ;; We can't have this character.
113 (call ccl-mule-utf-untrans
)
115 (r0 = -
1)))))))))))))))
117 (defconst utf-16le-decode-loop
125 ((translate-character utf-translation-table-for-decode r0 r1
)
126 (write-multibyte-character r0 r1
)))
129 (defconst utf-16be-decode-loop
137 ((translate-character utf-translation-table-for-decode r0 r1
)
138 (write-multibyte-character r0 r1
)))
143 (define-ccl-program ccl-decode-mule-utf-16le
144 `(2 ; 2 bytes -> 1 to 4 bytes
145 ,utf-16le-decode-loop
148 (call ccl-mule-utf-untrans
)))
152 (r0 = ,(charset-id 'eight-bit-control
))
153 (r0 = ,(charset-id 'eight-bit-graphic
)))
154 (write-multibyte-character r0 r3
)))))
155 "Decode UTF-16LE (little endian without signature bytes).
156 Basic decoding is done into the charsets ascii, latin-iso8859-1 and
157 mule-unicode-*. Un-representable Unicode characters are decoded as
158 U+fffd. The result is run through the translation-table named
159 `utf-translation-table-for-decode'.")
161 (define-ccl-program ccl-decode-mule-utf-16be
162 `(2 ; 2 bytes -> 1 to 4 bytes
163 ,utf-16be-decode-loop
166 (call ccl-mule-utf-untrans
)))
169 (call ccl-mule-utf-untrans
)))))
170 "Decode UTF-16BE (big endian without signature bytes).
171 Basic decoding is done into the charsets ascii, latin-iso8859-1 and
172 mule-unicode-*. Un-representable Unicode characters are
173 decoded as U+fffd. The result is run through the translation-table of
174 name `utf-translation-table-for-decode'.")
176 (define-ccl-program ccl-decode-mule-utf-16le-with-signature
180 ,@utf-16le-decode-loop
)
183 (call ccl-mule-utf-untrans
))))
184 "Like ccl-decode-utf-16le but skip the first 2-byte BOM.")
186 (define-ccl-program ccl-decode-mule-utf-16be-with-signature
190 ,@utf-16be-decode-loop
)
193 (call ccl-mule-utf-untrans
))))
194 "Like ccl-decode-utf-16be but skip the first 2-byte BOM.")
196 (define-ccl-program ccl-decode-mule-utf-16
203 ;; R1 is a BOM for little endian. We keep this character as
204 ;; is temporarily. It is removed by post-read-conversion
207 (write-multibyte-character r0 r1
)
208 ,@utf-16le-decode-loop
)
210 ;; R1 is a BOM for big endian, but we can't keep that
211 ;; character in the output because it can't be
212 ;; distinguished with the normal U+FEFF. So, we keep
216 (write-multibyte-character r0 r1
))
217 ;; R1 is a normal Unicode character.
220 ((translate-character utf-translation-table-for-decode r0 r1
)
221 (write-multibyte-character r0 r1
)))))
222 ,@utf-16be-decode-loop
)))
225 (call ccl-mule-utf-untrans
))))
226 "Like ccl-decode-utf-16be/le but check the first BOM.")
228 (makunbound 'utf-16-decode-ucs
) ; done with it
229 (makunbound 'utf-16le-decode-loop
)
230 (makunbound 'utf-16be-decode-loop
)
232 ;; UTF-16 decoder generates an UTF-8 sequence represented by a
233 ;; sequence eight-bit-control/graphic chars for an invalid byte (the
234 ;; last byte of an odd length source) and an untranslatable character
235 ;; (including an invalid surrogate-pair code-point).
237 ;; This CCL parses that sequence (the first byte is already in r1),
238 ;; and if the sequence represents an untranslatable character, it sets
239 ;; r1 to the original invalid code or untranslated Unicode character
240 ;; code, sets r2 to -1 (to prevent r2 and r3 are written), set2 r5 to
241 ;; -1 (to tell the caller that there's no pre-read character).
243 ;; If the sequence represents an invalid byte, it sets r1 to -1, r2 to
244 ;; the byte, sets r3 and r5 to -1.
246 ;; Otherwise, don't change r1, set r2 and r3 to already read
247 ;; eight-bit-control/graphic characters (if any), set r5 and r6 to the
248 ;; last character that invalidates the UTF-8 form.
250 ;; Note: For UTF-8 validation, we only check if a character is
251 ;; eight-bit-control/graphic or not. It may result in incorrect
252 ;; handling of random binary data, but such a data can't be encoded by
253 ;; UTF-16 anyway. At least, UTF-16 decoder doesn't generate such a
254 ;; sequence even if a source contains invalid byte-sequence.
256 (define-ccl-program ccl-mule-utf-16-encode-untrans
259 ;; Read the 2nd byte.
260 (read-multibyte-character r5 r6
)
261 (r0 = (r5 != ,(charset-id 'eight-bit-control
)))
262 (if ((r5 != ,(charset-id 'eight-bit-graphic
)) & r0
)
266 (end))) ; invalid UTF-8
271 ;; 2-byte UTF-8, i.e. originally an invalid byte.
273 (r2 |
= ((r1 & #x1F
) << 6))
278 ;; Read the 3rd byte.
279 (read-multibyte-character r5 r6
)
280 (r0 = (r5 != ,(charset-id 'eight-bit-control
)))
281 (if ((r5 != ,(charset-id 'eight-bit-graphic
)) & r0
)
282 ((end))) ; invalid UTF-8
284 (if (r1 < #xF0
) ; valid 3-byte UTF-8
285 ((r1 = ((r1 & #x0F
) << 12))
286 (r1 |
= ((r2 & #x3F
) << 6))
293 ;; Read the 4th byte.
294 (read-multibyte-character r5 r6
)
295 (r0 = (r5 != ,(charset-id 'eight-bit-control
)))
296 (if ((r5 != ,(charset-id 'eight-bit-graphic
)) & r0
)
297 (end)) ; livalid UTF-8
299 ;; valid 4-byte UTF-8
300 (r1 = ((r1 & #x07
) << 18))
301 (r1 |
= ((r2 & #x3F
) << 12))
302 (r1 |
= ((r3 & #x3F
) << 6))
316 (defconst utf-16-decode-to-ucs
317 ;; Read a character and set r1 to the corresponding Unicode code.
318 ;; If r5 is not negative, it means that we have already read a
319 ;; character into r5 and r6.
320 ;; If an invalid eight-bit-control/graphic sequence is found, r2 and
321 ;; r3 may contain a byte to written out, r5 and r6 may contain a
322 ;; pre-read character. Usually they are set to -1.
324 (read-multibyte-character r0 r1
)
328 (lookup-character utf-subst-table-for-encode r0 r1
)
332 ((translate-character utf-translation-table-for-encode r0 r1
)
333 (if (r0 == ,(charset-id 'ascii
))
335 (if (r0 == ,(charset-id 'latin-iso8859-1
))
337 (if (r0 == ,(charset-id 'eight-bit-control
))
339 (if (r0 == ,(charset-id 'eight-bit-graphic
))
340 (call ccl-mule-utf-16-encode-untrans
)
341 ((r2 = ((r1 & #x7f
) -
32))
342 (r3 = ((((r1 >> 7) -
32) * 96) + r2
))
345 (if (r0 == ,(charset-id 'mule-unicode-0100-24ff
))
347 (if (r0 == ,(charset-id 'mule-unicode-2500-33ff
))
349 (if (r0 == ,(charset-id 'mule-unicode-e000-ffff
))
351 (r1 = #xfffd
)))))))))))))
353 (defconst utf-16le-encode-loop
356 ,@utf-16-decode-to-ucs
359 (r0 = ((r1 >> 10) + #xD800
))
362 (r1 = ((r1 & #x3FF
) + #xDC00
))))
372 (defconst utf-16be-encode-loop
375 ,@utf-16-decode-to-ucs
378 (r0 = ((r1 >> 10) + #xD800
))
381 (r1 = ((r1 & #x3FF
) + #xDC00
))))
393 (define-ccl-program ccl-encode-mule-utf-16le
395 ,utf-16le-encode-loop
)
396 "Encode to UTF-16LE (little endian without signature).
397 Characters from the charsets ascii, eight-bit-control,
398 eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
399 after translation through the translation-table of name
400 `utf-translation-table-for-encode'.
401 Others are encoded as U+FFFD.")
403 (define-ccl-program ccl-encode-mule-utf-16be
405 ,utf-16be-encode-loop
)
406 "Encode to UTF-16BE (big endian without signature).
407 Characters from the charsets ascii, eight-bit-control,
408 eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
409 after translation through the translation-table named
410 `utf-translation-table-for-encode'.
411 Others are encoded as U+FFFD.")
413 (define-ccl-program ccl-encode-mule-utf-16le-with-signature
417 ,@utf-16le-encode-loop
))
418 "Encode to UTF-16 (little endian with signature).
419 Characters from the charsets ascii, eight-bit-control,
420 eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
421 after translation through the translation-table of name
422 `utf-translation-table-for-encode'.
423 Others are encoded as U+FFFD.")
425 (define-ccl-program ccl-encode-mule-utf-16be-with-signature
429 ,@utf-16be-encode-loop
))
430 "Encode to UTF-16 (big endian with signature).
431 Characters from the charsets ascii, eight-bit-control,
432 eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
433 after translation through the translation-table named
434 `utf-translation-table-for-encode'.
435 Others are encoded as U+FFFD.")
437 (makunbound 'utf-16-decode-to-ucs
)
438 (makunbound 'utf-16le-encode-loop
)
439 (makunbound 'utf-16be-encode-loop
)
441 (defun mule-utf-16-post-read-conversion (length)
443 (setq length
(utf-8-post-read-conversion length
))
444 (let ((char (following-char)))
445 (cond ((= char
(decode-char 'ucs
#xFFFE
))
447 (setq last-coding-system-used
448 (coding-system-change-text-conversion
449 last-coding-system-used
450 'mule-utf-16le-with-signature
))
451 (setq length
(1- length
)))
452 ((= char
(decode-char 'ucs
#xFFFF
))
454 (setq last-coding-system-used
455 (coding-system-change-text-conversion
456 last-coding-system-used
457 'mule-utf-16be-with-signature
))
458 (setq length
(1- length
)))
460 (setq last-coding-system-used
'mule-utf-16be
)))))
465 It supports Unicode characters of these ranges:
466 U+0000..U+33FF, U+E000..U+FFFF.
467 They correspond to these Emacs character sets:
468 ascii, latin-iso8859-1, mule-unicode-0100-24ff,
469 mule-unicode-2500-33ff, mule-unicode-e000-ffff
471 On decoding (e.g. reading a file), Unicode characters not in the above
472 ranges are decoded as U+FFFD, effectively corrupting the data
473 if they are re-encoded.
475 On encoding (e.g. writing a file), Emacs characters not belonging to
476 any of the character sets listed above are encoded into the byte
477 sequence representing U+FFFD (REPLACEMENT CHARACTER).")
478 (props `((safe-charsets
483 mule-unicode-0100-24ff
484 mule-unicode-2500-33ff
485 mule-unicode-e000-ffff
486 ,@(if utf-translate-cjk-mode
487 utf-translate-cjk-charsets
))
488 (valid-codes (0 .
255))
489 (mime-text-unsuitable . t
)
490 (pre-write-conversion . utf-8-pre-write-conversion
)
491 (dependency unify-8859-on-encoding-mode
492 unify-8859-on-decoding-mode
493 utf-fragment-on-decoding
494 utf-translate-cjk-mode
))))
497 ?u
; Mule-UCS uses ?U, but code-pages uses that for koi8-u.
499 "UTF-16LE encoding for Emacs-supported Unicode characters."
501 '(ccl-decode-mule-utf-16le . ccl-encode-mule-utf-16le
)
503 (post-read-conversion . utf-8-post-read-conversion
)
504 (mime-charset . utf-16le
)))
509 "UTF-16BE encoding for Emacs-supported Unicode characters."
512 '(ccl-decode-mule-utf-16be . ccl-encode-mule-utf-16be
)
514 (post-read-conversion . utf-8-post-read-conversion
)
515 (mime-charset . utf-16be
)))
518 'mule-utf-16le-with-signature
4 ?u
520 "Little endian UTF-16 (with BOM) for Emacs-supported Unicode characters."
523 '(ccl-decode-mule-utf-16le-with-signature
524 . ccl-encode-mule-utf-16le-with-signature
)
526 (post-read-conversion . utf-8-post-read-conversion
)
527 (coding-category . coding-category-utf-16-le
)
528 (mime-charset . utf-16
)))
531 'mule-utf-16be-with-signature
4 ?u
533 "Big endian UTF-16 (with BOM) for Emacs-supported Unicode characters."
536 '(ccl-decode-mule-utf-16be-with-signature
537 . ccl-encode-mule-utf-16be-with-signature
)
539 (post-read-conversion . utf-8-post-read-conversion
)
540 (coding-category . coding-category-utf-16-be
)
541 (mime-charset . utf-16
)))
546 "UTF-16 (with or without BOM) for Emacs-supported Unicode characters."
549 '(ccl-decode-mule-utf-16 . ccl-encode-mule-utf-16be-with-signature
)
551 (post-read-conversion . mule-utf-16-post-read-conversion
)
552 (coding-category . coding-category-utf-16-be
)
553 (mime-charset . utf-16
)))
556 (define-coding-system-alias 'utf-16le
'mule-utf-16le
)
557 (define-coding-system-alias 'utf-16be
'mule-utf-16be
)
558 (define-coding-system-alias 'utf-16le-with-signature
559 'mule-utf-16le-with-signature
)
560 (define-coding-system-alias 'utf-16be-with-signature
561 'mule-utf-16be-with-signature
)
562 (define-coding-system-alias 'utf-16
'mule-utf-16
)
564 ;; For backward compatibility.
565 (define-coding-system-alias 'mule-utf-16-le
'mule-utf-16le-with-signature
)
566 (define-coding-system-alias 'utf-16-le
'mule-utf-16le-with-signature
)
567 (define-coding-system-alias 'mule-utf-16-be
'mule-utf-16be-with-signature
)
568 (define-coding-system-alias 'utf-16-be
'mule-utf-16be-with-signature
)
570 ;;; arch-tag: 85455d46-d9c9-466d-a6f3-c3582a7367c4
571 ;;; utf-16.el ends here