Copy updated files from lib9/utf from upstream repository. This also requires updating code that previously took a shortcut and depended on the Rune and char32 typedefs to be identical (which is no longer the case, as Rune is now unsigned). R=jia.shao.peng@gmail.com BUG= Review URL: https://codereview.appspot.com/136920043 git-svn-id: http://libphonenumber.googlecode.com/svn/trunk/cpp/src/phonenumbers@699 ee073f10-1060-11df-b6a4-87a95322a99c
diff --git a/utf/rune.c b/utf/rune.c index c268489..b4aa93b 100644 --- a/utf/rune.c +++ b/utf/rune.c
@@ -1,6 +1,7 @@ /* * The authors of this software are Rob Pike and Ken Thompson. * Copyright (c) 2002 by Lucent Technologies. + * Portions Copyright (c) 2009 The Go Authors. All rights reserved. * Permission to use, copy, modify, and distribute this software for any * purpose without fee is hereby granted, provided that this entire notice * is included in all copies of any software which is or includes a copy @@ -11,8 +12,6 @@ * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE MERCHANTABILITY * OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. */ -#include <stdarg.h> -#include <string.h> #include "phonenumbers/utf/utf.h" #include "phonenumbers/utf/utfdef.h" @@ -35,12 +34,14 @@ Rune1 = (1<<(Bit1+0*Bitx))-1, /* 0000 0000 0111 1111 */ Rune2 = (1<<(Bit2+1*Bitx))-1, /* 0000 0111 1111 1111 */ Rune3 = (1<<(Bit3+2*Bitx))-1, /* 1111 1111 1111 1111 */ - Rune4 = (1<<(Bit4+3*Bitx))-1, - /* 0001 1111 1111 1111 1111 1111 */ + Rune4 = (1<<(Bit4+3*Bitx))-1, /* 0001 1111 1111 1111 1111 1111 */ Maskx = (1<<Bitx)-1, /* 0011 1111 */ Testx = Maskx ^ 0xFF, /* 1100 0000 */ + SurrogateMin = 0xD800, + SurrogateMax = 0xDFFF, + Bad = Runeerror, }; @@ -79,7 +80,7 @@ */ c = *(uchar*)str; if(c < Tx) { - *rune = c; + *rune = (Rune)c; return 1; } @@ -101,7 +102,7 @@ l = ((c << Bitx) | c1) & Rune2; if(l <= Rune1) goto bad; - *rune = l; + *rune = (Rune)l; return 2; } @@ -121,7 +122,9 @@ l = ((((c << Bitx) | c1) << Bitx) | c2) & Rune3; if(l <= Rune2) goto bad; - *rune = l; + if (SurrogateMin <= l && l <= SurrogateMax) + goto bad; + *rune = (Rune)l; return 3; } @@ -137,9 +140,9 @@ goto bad; if (c < T5) { l = ((((((c << Bitx) | c1) << Bitx) | c2) << Bitx) | c3) & Rune4; - if (l <= Rune3) + if (l <= Rune3 || l > Runemax) goto bad; - *rune = l; + *rune = (Rune)l; return 4; } @@ -175,7 +178,7 @@ */ c = *(uchar*)str; if(c < Tx) { - *rune = c; + *rune = (Rune)c; return 1; } @@ -192,7 +195,7 @@ l = ((c << Bitx) | c1) & Rune2; if(l <= Rune1) goto bad; - *rune = l; + *rune = (Rune)l; return 2; } @@ -207,7 +210,9 @@ l = ((((c << Bitx) | c1) << Bitx) | c2) & Rune3; if(l <= Rune2) goto bad; - *rune = l; + if (SurrogateMin <= l && l <= SurrogateMax) + goto bad; + *rune = (Rune)l; return 3; } @@ -220,9 +225,9 @@ goto bad; if (c < T5) { l = ((((((c << Bitx) | c1) << Bitx) | c2) << Bitx) | c3) & Rune4; - if (l <= Rune3) + if (l <= Rune3 || l > Runemax) goto bad; - *rune = l; + *rune = (Rune)l; return 4; } @@ -240,7 +245,8 @@ } int -isvalidcharntorune(const char* str, int length, Rune* rune, int* consumed) { +isvalidcharntorune(const char* str, int length, Rune* rune, int* consumed) +{ *consumed = charntorune(rune, str, length); return *rune != Runeerror || *consumed == 3; } @@ -257,7 +263,7 @@ */ c = *rune; if(c <= Rune1) { - str[0] = c; + str[0] = (char)c; return 1; } @@ -266,28 +272,30 @@ * 0080-07FF => T2 Tx */ if(c <= Rune2) { - str[0] = T2 | (c >> 1*Bitx); - str[1] = Tx | (c & Maskx); + str[0] = (char)(T2 | (c >> 1*Bitx)); + str[1] = (char)(Tx | (c & Maskx)); return 2; } /* - * If the Rune is out of range, convert it to the error rune. + * If the Rune is out of range or a surrogate half, convert it to the error rune. * Do this test here because the error rune encodes to three bytes. * Doing it earlier would duplicate work, since an out of range * Rune wouldn't have fit in one or two bytes. */ if (c > Runemax) c = Runeerror; + if (SurrogateMin <= c && c <= SurrogateMax) + c = Runeerror; /* * three character sequence * 0800-FFFF => T3 Tx Tx */ if (c <= Rune3) { - str[0] = T3 | (c >> 2*Bitx); - str[1] = Tx | ((c >> 1*Bitx) & Maskx); - str[2] = Tx | (c & Maskx); + str[0] = (char)(T3 | (c >> 2*Bitx)); + str[1] = (char)(Tx | ((c >> 1*Bitx) & Maskx)); + str[2] = (char)(Tx | (c & Maskx)); return 3; } @@ -295,10 +303,10 @@ * four character sequence (21-bit value) * 10000-1FFFFF => T4 Tx Tx Tx */ - str[0] = T4 | (c >> 3*Bitx); - str[1] = Tx | ((c >> 2*Bitx) & Maskx); - str[2] = Tx | ((c >> 1*Bitx) & Maskx); - str[3] = Tx | (c & Maskx); + str[0] = (char)(T4 | (c >> 3*Bitx)); + str[1] = (char)(Tx | ((c >> 2*Bitx) & Maskx)); + str[2] = (char)(Tx | ((c >> 1*Bitx) & Maskx)); + str[3] = (char)(Tx | (c & Maskx)); return 4; } @@ -317,7 +325,7 @@ nb = 0; while(nrune--) { - c = *r++; + c = (int)*r++; if (c <= Rune1) nb++; else if (c <= Rune2)
diff --git a/utf/unicodetext.cc b/utf/unicodetext.cc index 55ffedf..ecd3230 100644 --- a/utf/unicodetext.cc +++ b/utf/unicodetext.cc
@@ -85,7 +85,7 @@ } } // Is the current string invalid UTF8 or just non-interchange UTF8? - char32 rune; + Rune rune; int n; if (isvalidcharntorune(start, end - start, &rune, &n)) { // structurally valid UTF8, but not interchange valid @@ -362,7 +362,8 @@ void UnicodeText::push_back(char32 c) { if (UniLib::IsValidCodepoint(c)) { char buf[UTFmax]; - int len = runetochar(buf, &c); + Rune rune = c; + int len = runetochar(buf, &rune); if (UniLib::IsInterchangeValid(buf, len)) { repr_.append(buf, len); } else {
diff --git a/utf/unilib.cc b/utf/unilib.cc index ffcb8b0..918134e 100644 --- a/utf/unilib.cc +++ b/utf/unilib.cc
@@ -46,7 +46,7 @@ } // namespace int SpanInterchangeValid(const char* begin, int byte_length) { - char32 rune; + Rune rune; const char* p = begin; const char* end = begin + byte_length; while (p < end) {
diff --git a/utf/utf.h b/utf/utf.h index f4fd482..72d01ed 100644 --- a/utf/utf.h +++ b/utf/utf.h
@@ -1,27 +1,22 @@ /* * The authors of this software are Rob Pike and Ken Thompson. - * Copyright (c) 1998-2002 by Lucent Technologies. - * Portions Copyright (c) 2009 The Go Authors. All rights reserved. + * Copyright (c) 1998-2002 by Lucent Technologies. + * Portions Copyright (c) 2009 The Go Authors. All rights reserved. * Permission to use, copy, modify, and distribute this software for any * purpose without fee is hereby granted, provided that this entire notice * is included in all copies of any software which is or includes a copy * or modification of this software and in all copies of the supporting * documentation for such software. * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED - * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES MAKE ANY + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES MAKE ANY * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE MERCHANTABILITY * OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. - */ + */ #ifndef _UTFH_ #define _UTFH_ 1 -// stdint.h content doesn't seem to be used in this file and doesn't exist on -// Windows, therefore we comment it out here so that the code could be compiled -// on Windows. -//#include <stdint.h> - -typedef signed int Rune; /* Code-point values in Unicode 4.0 are 21 bits wide.*/ +typedef unsigned int Rune; /* Code-point values in Unicode 4.0 are 21 bits wide.*/ enum { @@ -71,7 +66,7 @@ // n bytes of s. If the UTF sequence is incomplete within n bytes, // charntorune will set *r to Runeerror and return 0. If it is complete // but not in UTF format, it will set *r to Runeerror and return 1. -// +// // Added 2004-09-24 by Wei-Hwa Huang int charntorune(Rune* r, const char* s, int n); @@ -126,7 +121,7 @@ // byte terminating a string is considered to be part of the string s. // (cf. strchr) -const char* utfrune(const char* s, Rune r); +/*const*/ char* utfrune(const char* s, Rune r); // utfrrune returns a pointer to the last occurrence of rune r in the @@ -134,7 +129,7 @@ // byte terminating a string is considered to be part of the string s. // (cf. strrchr) -const char* utfrrune(const char* s, Rune r); +/*const*/ char* utfrrune(const char* s, Rune r); // utfutf returns a pointer to the first occurrence of the UTF string @@ -155,7 +150,7 @@ // These functions are rune-string analogues of the corresponding // functions in strcat (3). -// +// // These routines first appeared in Plan 9. // SEE ALSO // memmove (3) @@ -208,8 +203,8 @@ // isupperrune tests for upper case characters, including Unicode // upper case letters and targets of the toupper mapping. islowerrune -// and istitlerune are defined analogously. - +// and istitlerune are defined analogously. + int isupperrune(Rune r); int islowerrune(Rune r); int istitlerune(Rune r); @@ -227,12 +222,6 @@ int isdigitrune(Rune r); -// isideographicrune tests for ideographic characters and numbers, as -// defined by the Unicode standard. - -int isideographicrune(Rune r); - - // isspacerune tests for whitespace characters, including "C" locale // whitespace, Unicode defined whitespace, and the "zero-width // non-break space" character.
diff --git a/utf/utfdef.h b/utf/utfdef.h index adc6d95..4bbdfc6 100644 --- a/utf/utfdef.h +++ b/utf/utfdef.h
@@ -25,4 +25,3 @@ typedef unsigned long ulong; #define nelem(x) (sizeof(x)/sizeof((x)[0])) -#define nil ((void*)0)