From 4bd327a4cc0a001a796413148e1fbded36fb6ec8 Mon Sep 17 00:00:00 2001
From: Karel Miko <karel.miko@gmail.com>
Date: Tue, 28 Feb 2017 16:29:25 +0100
Subject: [PATCH] better handling wchar_t when __WCHAR_MAX__ is 0xFFFF (2 bytes
 only)

---
 src/headers/tomcrypt_pk.h                     |  1 +
 src/pk/asn1/der/utf8/der_encode_utf8_string.c |  4 +--
 src/pk/asn1/der/utf8/der_length_utf8_string.c | 26 ++++++++++++++++---
 3 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/src/headers/tomcrypt_pk.h b/src/headers/tomcrypt_pk.h
index c457ce8..cd3d021 100644
--- a/src/headers/tomcrypt_pk.h
+++ b/src/headers/tomcrypt_pk.h
@@ -619,6 +619,7 @@ int der_encode_utf8_string(const wchar_t *in,  unsigned long inlen,
 int der_decode_utf8_string(const unsigned char *in,  unsigned long inlen,
                                        wchar_t *out, unsigned long *outlen);
 unsigned long der_utf8_charsize(const wchar_t c);
+int der_utf8_valid_char(const wchar_t c);
 int der_length_utf8_string(const wchar_t *in, unsigned long noctets, unsigned long *outlen);
 
 
diff --git a/src/pk/asn1/der/utf8/der_encode_utf8_string.c b/src/pk/asn1/der/utf8/der_encode_utf8_string.c
index 7a782f1..ab7ab54 100644
--- a/src/pk/asn1/der/utf8/der_encode_utf8_string.c
+++ b/src/pk/asn1/der/utf8/der_encode_utf8_string.c
@@ -37,9 +37,7 @@ int der_encode_utf8_string(const wchar_t *in,  unsigned long inlen,
 
    /* get the size */
    for (x = len = 0; x < inlen; x++) {
-       if (in[x] < 0 || in[x] > 0x1FFFF) {
-          return CRYPT_INVALID_ARG;
-       }
+       if (!der_utf8_valid_char(in[x])) return CRYPT_INVALID_ARG;
        len += der_utf8_charsize(in[x]);
    }
 
diff --git a/src/pk/asn1/der/utf8/der_length_utf8_string.c b/src/pk/asn1/der/utf8/der_length_utf8_string.c
index 2ce2ca4..1135acc 100644
--- a/src/pk/asn1/der/utf8/der_length_utf8_string.c
+++ b/src/pk/asn1/der/utf8/der_length_utf8_string.c
@@ -27,11 +27,33 @@ unsigned long der_utf8_charsize(const wchar_t c)
       return 1;
    } else if (c <= 0x7FF) {
       return 2;
+#if __WCHAR_MAX__ == 0xFFFF
+   } else {
+      return 3;
+   }
+#else
    } else if (c <= 0xFFFF) {
       return 3;
    } else {
       return 4;
    }
+#endif
+}
+
+/**
+  Test whether the given code point is valid character
+  @param c   The UTF-8 character to test
+  @return    1 - valid, 0 - invalid
+*/
+int der_utf8_valid_char(const wchar_t c)
+{
+#if !defined(__WCHAR_MAX__) || __WCHAR_MAX__ > 0xFFFF
+   if (in[x] > 0x10FFFF) return 0;
+#endif
+#if !defined(__WCHAR_MAX__) || __WCHAR_MAX__ != 0xFFFF && __WCHAR_MAX__ != 0xFFFFFFFF
+   if (in[x] < 0) return 0;
+#endif
+   return 1;
 }
 
 /**
@@ -50,9 +72,7 @@ int der_length_utf8_string(const wchar_t *in, unsigned long noctets, unsigned lo
 
    len = 0;
    for (x = 0; x < noctets; x++) {
-      if (in[x] < 0 || in[x] > 0x10FFFF) {
-         return CRYPT_INVALID_ARG;
-      }
+      if (!der_utf8_valid_char(in[x])) return CRYPT_INVALID_ARG;
       len += der_utf8_charsize(in[x]);
    }