From fa42c37fe3359505d76a5e7416728f6802705363 Mon Sep 17 00:00:00 2001 From: Kenichi Handa Date: Tue, 7 Mar 2000 06:17:54 +0000 Subject: [PATCH] Add comments on coding-category-utf-8, coding-category-utf-16-be, and coding-category-utf-16-le. (coding_category_name): Include "coding-category-utf-8", "coding-category-utf-16-be", and "coding-category-utf-16-le". (UTF_8_1_OCTET_P) (UTF_8_EXTRA_OCTET_P) (UTF_8_2_OCTET_LEADING_P) (UTF_8_3_OCTET_LEADING_P) (UTF_8_4_OCTET_LEADING_P) (UTF_8_5_OCTET_LEADING_P) (UTF_8_6_OCTET_LEADING_P): New macros. (detect_coding_utf_8): New function. (UTF_16_INVALID_P) (TF_16_HIGH_SURROGATE_P) (UTF_16_LOW_SURROGATE_P): New macros. (detect_coding_utf_16): New function. (detect_coding_mask): Fix bug of returning wrong mask bits in the case that detect_coding_XXX returns a mask not set in priorities[i]. (detect_eol_type_in_2_octet_form): New function. (detect_eol): If cooding->category_idx is for UTF-16, call detect_eol_type_in_2_octet_form instead of dectect_eol_type. (detect_coding_system): Don't include `nil' coding-system in the result. (Fupdate_coding_systems_internal): Update all coding-categories. --- src/coding.c | 299 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 258 insertions(+), 41 deletions(-) diff --git a/src/coding.c b/src/coding.c index 40fe424fe0..fcd5e89e00 100644 --- a/src/coding.c +++ b/src/coding.c @@ -362,6 +362,9 @@ char *coding_category_name[CODING_CATEGORY_IDX_MAX] = { "coding-category-iso-8-else", "coding-category-ccl", "coding-category-big5", + "coding-category-utf-8", + "coding-category-utf-16-be", + "coding-category-utf-16-le", "coding-category-raw-text", "coding-category-binary" }; @@ -2348,6 +2351,89 @@ detect_coding_big5 (src, src_end) return CODING_CATEGORY_MASK_BIG5; } +/* See the above "GENERAL NOTES on `detect_coding_XXX ()' functions". + Check if a text is encoded in UTF-8. If it is, return + CODING_CATEGORY_MASK_UTF_8, else return 0. */ + +#define UTF_8_1_OCTET_P(c) ((c) < 0x80) +#define UTF_8_EXTRA_OCTET_P(c) (((c) & 0xC0) == 0x80) +#define UTF_8_2_OCTET_LEADING_P(c) (((c) & 0xE0) == 0xC0) +#define UTF_8_3_OCTET_LEADING_P(c) (((c) & 0xF0) == 0xE0) +#define UTF_8_4_OCTET_LEADING_P(c) (((c) & 0xF8) == 0xF0) +#define UTF_8_5_OCTET_LEADING_P(c) (((c) & 0xFC) == 0xF8) +#define UTF_8_6_OCTET_LEADING_P(c) (((c) & 0xFE) == 0xFC) + +int +detect_coding_utf_8 (src, src_end) + unsigned char *src, *src_end; +{ + unsigned char c; + int seq_maybe_bytes; + + while (src < src_end) + { + c = *src++; + if (UTF_8_1_OCTET_P (c)) + continue; + else if (UTF_8_2_OCTET_LEADING_P (c)) + seq_maybe_bytes = 1; + else if (UTF_8_3_OCTET_LEADING_P (c)) + seq_maybe_bytes = 2; + else if (UTF_8_4_OCTET_LEADING_P (c)) + seq_maybe_bytes = 3; + else if (UTF_8_5_OCTET_LEADING_P (c)) + seq_maybe_bytes = 4; + else if (UTF_8_6_OCTET_LEADING_P (c)) + seq_maybe_bytes = 5; + else + return 0; + + do + { + if (src >= src_end) + return CODING_CATEGORY_MASK_UTF_8; + + c = *src++; + if (!UTF_8_EXTRA_OCTET_P (c)) + return 0; + seq_maybe_bytes--; + } + while (seq_maybe_bytes > 0); + } + + return CODING_CATEGORY_MASK_UTF_8; +} + +/* See the above "GENERAL NOTES on `detect_coding_XXX ()' functions". + Check if a text is encoded in UTF-16 Big Endian (endian == 1) or + Little Endian (otherwise). If it is, return + CODING_CATEGORY_MASK_UTF_16_BE or CODING_CATEGORY_MASK_UTF_16_LE, + else return 0. */ + +#define UTF_16_INVALID_P(val) \ + (((val) == 0xFFFE) \ + || ((val) == 0xFFFF)) + +#define UTF_16_HIGH_SURROGATE_P(val) \ + (((val) & 0xD800) == 0xD800) + +#define UTF_16_LOW_SURROGATE_P(val) \ + (((val) & 0xDC00) == 0xDC00) + +int +detect_coding_utf_16 (src, src_end) + unsigned char *src, *src_end; +{ + if ((src + 1) >= src_end) return 0; + + if ((src[0] == 0xFF) && (src[1] == 0xFE)) + return CODING_CATEGORY_MASK_UTF_16_LE; + else if ((src[0] == 0xFE) && (src[1] == 0xFF)) + return CODING_CATEGORY_MASK_UTF_16_BE; + + return 0; +} + /* See the above "GENERAL NOTES on `decode_coding_XXX ()' functions". If SJIS_P is 1, decode SJIS text, else decode BIG5 test. */ @@ -3453,6 +3539,26 @@ setup_raw_text_coding_system (coding) as BIG5. Assigned the coding-system (Lisp symbol) `cn-big5' by default. + o coding-category-utf-8 + + The category for a coding system which has the same code range + as UTF-8 (cf. RFC2279). Assigned the coding-system (Lisp + symbol) `utf-8' by default. + + o coding-category-utf-16-be + + The category for a coding system in which a text has an + Unicode signature (cf. Unicode Standard) in the order of BIG + endian at the head. Assigned the coding-system (Lisp symbol) + `utf-16-be' by default. + + o coding-category-utf-16-le + + The category for a coding system in which a text has an + Unicode signature (cf. Unicode Standard) in the order of + LITTLE endian at the head. Assigned the coding-system (Lisp + symbol) `utf-16-le' by default. + o coding-category-ccl The category for a coding system of which encoder/decoder is @@ -3481,7 +3587,10 @@ int ascii_skip_code[256]; /* Detect how a text of length SRC_BYTES pointed by SOURCE is encoded. If it detects possible coding systems, return an integer in which appropriate flag bits are set. Flag bits are defined by macros - CODING_CATEGORY_MASK_XXX in `coding.h'. + CODING_CATEGORY_MASK_XXX in `coding.h'. If PRIORITIES is non-NULL, + it should point the table `coding_priorities'. In that case, only + the flag bit for a coding system of the highest priority is set in + the returned value. How many ASCII characters are at the head is returned as *SKIP. */ @@ -3492,8 +3601,8 @@ detect_coding_mask (source, src_bytes, priorities, skip) { register unsigned char c; unsigned char *src = source, *src_end = source + src_bytes; - unsigned int mask; - int i; + unsigned int mask, utf16_examined_p, iso2022_examined_p; + int i, idx; /* At first, skip all ASCII characters and control characters except for three ISO2022 specific control characters. */ @@ -3528,7 +3637,14 @@ detect_coding_mask (source, src_bytes, priorities, skip) goto label_loop_detect_coding; } if (priorities) - goto label_return_highest_only; + { + for (i = 0; i < CODING_CATEGORY_IDX_MAX; i++) + { + if (mask & priorities[i]) + return priorities[i]; + } + return CODING_CATEGORY_MASK_RAW_TEXT; + } } else { @@ -3537,8 +3653,12 @@ detect_coding_mask (source, src_bytes, priorities, skip) if (c < 0xA0) { /* C is the first byte of SJIS character code, - or a leading-code of Emacs' internal format (emacs-mule). */ - try = CODING_CATEGORY_MASK_SJIS | CODING_CATEGORY_MASK_EMACS_MULE; + or a leading-code of Emacs' internal format (emacs-mule), + or the first byte of UTF-16. */ + try = (CODING_CATEGORY_MASK_SJIS + | CODING_CATEGORY_MASK_EMACS_MULE + | CODING_CATEGORY_MASK_UTF_16_BE + | CODING_CATEGORY_MASK_UTF_16_LE); /* Or, if C is a special latin extra code, or is an ISO2022 specific control code of C1 (SS2 or SS3), @@ -3559,11 +3679,15 @@ detect_coding_mask (source, src_bytes, priorities, skip) else /* C is a character of ISO2022 in graphic plane right, or a SJIS's 1-byte character code (i.e. JISX0201), - or the first byte of BIG5's 2-byte code. */ + or the first byte of BIG5's 2-byte code, + or the first byte of UTF-8/16. */ try = (CODING_CATEGORY_MASK_ISO_8_ELSE | CODING_CATEGORY_MASK_ISO_8BIT | CODING_CATEGORY_MASK_SJIS - | CODING_CATEGORY_MASK_BIG5); + | CODING_CATEGORY_MASK_BIG5 + | CODING_CATEGORY_MASK_UTF_8 + | CODING_CATEGORY_MASK_UTF_16_BE + | CODING_CATEGORY_MASK_UTF_16_LE); /* Or, we may have to consider the possibility of CCL. */ if (coding_system_table[CODING_CATEGORY_IDX_CCL] @@ -3572,26 +3696,40 @@ detect_coding_mask (source, src_bytes, priorities, skip) try |= CODING_CATEGORY_MASK_CCL; mask = 0; + utf16_examined_p = iso2022_examined_p = 0; if (priorities) { for (i = 0; i < CODING_CATEGORY_IDX_MAX; i++) { - if (priorities[i] & try & CODING_CATEGORY_MASK_ISO) - mask = detect_coding_iso2022 (src, src_end); + if (!iso2022_examined_p + && (priorities[i] & try & CODING_CATEGORY_MASK_ISO)) + { + mask |= detect_coding_iso2022 (src, src_end); + iso2022_examined_p = 1; + } else if (priorities[i] & try & CODING_CATEGORY_MASK_SJIS) - mask = detect_coding_sjis (src, src_end); + mask |= detect_coding_sjis (src, src_end); + else if (priorities[i] & try & CODING_CATEGORY_MASK_UTF_8) + mask |= detect_coding_utf_8 (src, src_end); + else if (!utf16_examined_p + && (priorities[i] & try & + CODING_CATEGORY_MASK_UTF_16_BE_LE)) + { + mask |= detect_coding_utf_16 (src, src_end); + utf16_examined_p = 1; + } else if (priorities[i] & try & CODING_CATEGORY_MASK_BIG5) - mask = detect_coding_big5 (src, src_end); + mask |= detect_coding_big5 (src, src_end); else if (priorities[i] & try & CODING_CATEGORY_MASK_EMACS_MULE) - mask = detect_coding_emacs_mule (src, src_end); + mask |= detect_coding_emacs_mule (src, src_end); else if (priorities[i] & try & CODING_CATEGORY_MASK_CCL) - mask = detect_coding_ccl (src, src_end); + mask |= detect_coding_ccl (src, src_end); else if (priorities[i] & CODING_CATEGORY_MASK_RAW_TEXT) - mask = CODING_CATEGORY_MASK_RAW_TEXT; + mask |= CODING_CATEGORY_MASK_RAW_TEXT; else if (priorities[i] & CODING_CATEGORY_MASK_BINARY) - mask = CODING_CATEGORY_MASK_BINARY; - if (mask) - goto label_return_highest_only; + mask |= CODING_CATEGORY_MASK_BINARY; + if (mask & priorities[i]) + return priorities[i]; } return CODING_CATEGORY_MASK_RAW_TEXT; } @@ -3601,20 +3739,16 @@ detect_coding_mask (source, src_bytes, priorities, skip) mask |= detect_coding_sjis (src, src_end); if (try & CODING_CATEGORY_MASK_BIG5) mask |= detect_coding_big5 (src, src_end); + if (try & CODING_CATEGORY_MASK_UTF_8) + mask |= detect_coding_utf_8 (src, src_end); + if (try & CODING_CATEGORY_MASK_UTF_16_BE_LE) + mask |= detect_coding_utf_16 (src, src_end); if (try & CODING_CATEGORY_MASK_EMACS_MULE) mask |= detect_coding_emacs_mule (src, src_end); if (try & CODING_CATEGORY_MASK_CCL) mask |= detect_coding_ccl (src, src_end); } return (mask | CODING_CATEGORY_MASK_RAW_TEXT | CODING_CATEGORY_MASK_BINARY); - - label_return_highest_only: - for (i = 0; i < CODING_CATEGORY_IDX_MAX; i++) - { - if (mask & priorities[i]) - return priorities[i]; - } - return CODING_CATEGORY_MASK_RAW_TEXT; } /* Detect how a text of length SRC_BYTES pointed by SRC is encoded. @@ -3710,6 +3844,76 @@ detect_eol_type (source, src_bytes, skip) return eol_type; } +/* Like detect_eol_type, but detect EOL type in 2-octet + big-endian/little-endian format for coding systems utf-16-be and + utf-16-le. */ + +static int +detect_eol_type_in_2_octet_form (source, src_bytes, skip, big_endian_p) + unsigned char *source; + int src_bytes, *skip; +{ + unsigned char *src = source, *src_end = src + src_bytes; + unsigned int c1, c2; + int total = 0; /* How many end-of-lines are found so far. */ + int eol_type = CODING_EOL_UNDECIDED; + int this_eol_type; + int msb, lsb; + + if (big_endian_p) + msb = 0, lsb = 1; + else + msb = 1, lsb = 0; + + *skip = 0; + + while ((src + 1) < src_end && total < MAX_EOL_CHECK_COUNT) + { + c1 = (src[msb] << 8) | (src[lsb]); + src += 2; + + if (c1 == '\n' || c1 == '\r') + { + if (*skip == 0) + *skip = src - 2 - source; + total++; + if (c1 == '\n') + { + this_eol_type = CODING_EOL_LF; + } + else + { + if ((src + 1) >= src_end) + { + this_eol_type = CODING_EOL_CR; + } + else + { + c2 = (src[msb] << 8) | (src[lsb]); + if (c2 == '\n') + this_eol_type = CODING_EOL_CRLF, src += 2; + else + this_eol_type = CODING_EOL_CR; + } + } + + if (eol_type == CODING_EOL_UNDECIDED) + /* This is the first end-of-line. */ + eol_type = this_eol_type; + else if (eol_type != this_eol_type) + { + /* The found type is different from what found before. */ + eol_type = CODING_EOL_INCONSISTENT; + break; + } + } + } + + if (*skip == 0) + *skip = src_end - source; + return eol_type; +} + /* Detect how end-of-line of a text of length SRC_BYTES pointed by SRC is encoded. If it detects an appropriate format of end-of-line, it sets the information in *CODING. */ @@ -3722,7 +3926,20 @@ detect_eol (coding, src, src_bytes) { Lisp_Object val; int skip; - int eol_type = detect_eol_type (src, src_bytes, &skip); + int eol_type; + + switch (coding->category_idx) + { + case CODING_CATEGORY_IDX_UTF_16_BE: + eol_type = detect_eol_type_in_2_octet_form (src, src_bytes, &skip, 1); + break; + case CODING_CATEGORY_IDX_UTF_16_LE: + eol_type = detect_eol_type_in_2_octet_form (src, src_bytes, &skip, 0); + break; + default: + eol_type = detect_eol_type (src, src_bytes, &skip); + break; + } if (coding->heading_ascii > skip) coding->heading_ascii = skip; @@ -5216,13 +5433,17 @@ detect_coding_system (src, src_bytes, highest) /* At first, gather possible coding systems in VAL. */ val = Qnil; - for (tmp = Vcoding_category_list; !NILP (tmp); tmp = XCDR (tmp)) + for (tmp = Vcoding_category_list; CONSP (tmp); tmp = XCDR (tmp)) { - int idx - = XFASTINT (Fget (XCAR (tmp), Qcoding_category_index)); - if (coding_mask & (1 << idx)) + Lisp_Object category_val, category_index; + + category_index = Fget (XCAR (tmp), Qcoding_category_index); + category_val = Fsymbol_value (XCAR (tmp)); + if (!NILP (category_val) + && NATNUMP (category_index) + && (coding_mask & (1 << XFASTINT (category_index)))) { - val = Fcons (Fsymbol_value (XCAR (tmp)), val); + val = Fcons (category_val, val); if (highest) break; } @@ -5231,7 +5452,7 @@ detect_coding_system (src, src_bytes, highest) val = Fnreverse (val); /* Then, replace the elements with subsidiary coding systems. */ - for (tmp = val; !NILP (tmp); tmp = XCDR (tmp)) + for (tmp = val; CONSP (tmp); tmp = XCDR (tmp)) { if (eol_type != CODING_EOL_UNDECIDED && eol_type != CODING_EOL_INCONSISTENT) @@ -5712,17 +5933,13 @@ which is a list of all the arguments given to this function.") DEFUN ("update-coding-systems-internal", Fupdate_coding_systems_internal, Supdate_coding_systems_internal, 0, 0, 0, "Update internal database for ISO2022 and CCL based coding systems.\n\ -When values of the following coding categories are changed, you must\n\ -call this function:\n\ - coding-category-iso-7, coding-category-iso-7-tight,\n\ - coding-category-iso-8-1, coding-category-iso-8-2,\n\ - coding-category-iso-7-else, coding-category-iso-8-else,\n\ - coding-category-ccl") +When values of any coding categories are changed, you must\n\ +call this function") () { int i; - for (i = CODING_CATEGORY_IDX_ISO_7; i <= CODING_CATEGORY_IDX_CCL; i++) + for (i = CODING_CATEGORY_IDX_EMACS_MULE; i < CODING_CATEGORY_IDX_MAX; i++) { Lisp_Object val; @@ -5767,7 +5984,7 @@ This function is internal use only.") } /* If coding-category-list is valid and contains all coding categories, `i' should be CODING_CATEGORY_IDX_MAX now. If not, - the following code saves Emacs from craching. */ + the following code saves Emacs from crashing. */ while (i < CODING_CATEGORY_IDX_MAX) coding_priorities[i++] = CODING_CATEGORY_MASK_RAW_TEXT; -- 2.20.1