summaryrefslogtreecommitdiff
path: root/utf8.c
diff options
context:
space:
mode:
Diffstat (limited to 'utf8.c')
-rw-r--r--utf8.c56
1 files changed, 46 insertions, 10 deletions
diff --git a/utf8.c b/utf8.c
index eb78587504..3b42fadffd 100644
--- a/utf8.c
+++ b/utf8.c
@@ -4,6 +4,11 @@
/* This code is originally from http://www.cl.cam.ac.uk/~mgk25/ucs/ */
+static const char utf16_be_bom[] = {'\xFE', '\xFF'};
+static const char utf16_le_bom[] = {'\xFF', '\xFE'};
+static const char utf32_be_bom[] = {'\0', '\0', '\xFE', '\xFF'};
+static const char utf32_le_bom[] = {'\xFF', '\xFE', '\0', '\0'};
+
struct interval {
ucs_char_t first;
ucs_char_t last;
@@ -470,16 +475,17 @@ int utf8_fprintf(FILE *stream, const char *format, ...)
#else
typedef char * iconv_ibp;
#endif
-char *reencode_string_iconv(const char *in, size_t insz, iconv_t conv, size_t *outsz_p)
+char *reencode_string_iconv(const char *in, size_t insz, iconv_t conv,
+ size_t bom_len, size_t *outsz_p)
{
size_t outsz, outalloc;
char *out, *outpos;
iconv_ibp cp;
outsz = insz;
- outalloc = st_add(outsz, 1); /* for terminating NUL */
+ outalloc = st_add(outsz, 1 + bom_len); /* for terminating NUL */
out = xmalloc(outalloc);
- outpos = out;
+ outpos = out + bom_len;
cp = (iconv_ibp)in;
while (1) {
@@ -540,10 +546,44 @@ char *reencode_string_len(const char *in, size_t insz,
{
iconv_t conv;
char *out;
+ const char *bom_str = NULL;
+ size_t bom_len = 0;
if (!in_encoding)
return NULL;
+ /* UTF-16LE-BOM is the same as UTF-16 for reading */
+ if (same_utf_encoding("UTF-16LE-BOM", in_encoding))
+ in_encoding = "UTF-16";
+
+ /*
+ * For writing, UTF-16 iconv typically creates "UTF-16BE-BOM"
+ * Some users under Windows want the little endian version
+ *
+ * We handle UTF-16 and UTF-32 ourselves only if the platform does not
+ * provide a BOM (which we require), since we want to match the behavior
+ * of the system tools and libc as much as possible.
+ */
+ if (same_utf_encoding("UTF-16LE-BOM", out_encoding)) {
+ bom_str = utf16_le_bom;
+ bom_len = sizeof(utf16_le_bom);
+ out_encoding = "UTF-16LE";
+ } else if (same_utf_encoding("UTF-16BE-BOM", out_encoding)) {
+ bom_str = utf16_be_bom;
+ bom_len = sizeof(utf16_be_bom);
+ out_encoding = "UTF-16BE";
+#ifdef ICONV_OMITS_BOM
+ } else if (same_utf_encoding("UTF-16", out_encoding)) {
+ bom_str = utf16_be_bom;
+ bom_len = sizeof(utf16_be_bom);
+ out_encoding = "UTF-16BE";
+ } else if (same_utf_encoding("UTF-32", out_encoding)) {
+ bom_str = utf32_be_bom;
+ bom_len = sizeof(utf32_be_bom);
+ out_encoding = "UTF-32BE";
+#endif
+ }
+
conv = iconv_open(out_encoding, in_encoding);
if (conv == (iconv_t) -1) {
in_encoding = fallback_encoding(in_encoding);
@@ -553,9 +593,10 @@ char *reencode_string_len(const char *in, size_t insz,
if (conv == (iconv_t) -1)
return NULL;
}
-
- out = reencode_string_iconv(in, insz, conv, outsz);
+ out = reencode_string_iconv(in, insz, conv, bom_len, outsz);
iconv_close(conv);
+ if (out && bom_str && bom_len)
+ memcpy(out, bom_str, bom_len);
return out;
}
#endif
@@ -566,11 +607,6 @@ static int has_bom_prefix(const char *data, size_t len,
return data && bom && (len >= bom_len) && !memcmp(data, bom, bom_len);
}
-static const char utf16_be_bom[] = {'\xFE', '\xFF'};
-static const char utf16_le_bom[] = {'\xFF', '\xFE'};
-static const char utf32_be_bom[] = {'\0', '\0', '\xFE', '\xFF'};
-static const char utf32_le_bom[] = {'\xFF', '\xFE', '\0', '\0'};
-
int has_prohibited_utf_bom(const char *enc, const char *data, size_t len)
{
return (