utf8.[ch] et al: use char32_t and char16_t instead of int, int32_t, int16_t
rework C11 utf8.[ch] to use char32_t instead of uint32_t when referring to unicode chars, to make things more expressive. [ @zonque: * rebased to current master * use AC_CHECK_DECLS to detect availibility of char{16,32}_t * make utf8_encoded_to_unichar() return int ]
This commit is contained in:
parent
9766c16bd0
commit
c932fb71cc
|
@ -297,7 +297,8 @@ AC_SUBST(CAP_LIBS)
|
||||||
|
|
||||||
AC_CHECK_FUNCS([memfd_create])
|
AC_CHECK_FUNCS([memfd_create])
|
||||||
AC_CHECK_FUNCS([__secure_getenv secure_getenv])
|
AC_CHECK_FUNCS([__secure_getenv secure_getenv])
|
||||||
AC_CHECK_DECLS([gettid, pivot_root, name_to_handle_at, setns, getrandom, renameat2, kcmp, keyctl, key_serial_t, LO_FLAGS_PARTSCAN],
|
AC_CHECK_DECLS([gettid, pivot_root, name_to_handle_at, setns, getrandom, renameat2,
|
||||||
|
kcmp, keyctl, key_serial_t, char16_t, char32_t, LO_FLAGS_PARTSCAN],
|
||||||
[], [], [[
|
[], [], [[
|
||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
|
@ -119,7 +119,7 @@ char *cescape(const char *s) {
|
||||||
return cescape_length(s, strlen(s));
|
return cescape_length(s, strlen(s));
|
||||||
}
|
}
|
||||||
|
|
||||||
int cunescape_one(const char *p, size_t length, uint32_t *ret, bool *eight_bit) {
|
int cunescape_one(const char *p, size_t length, char32_t *ret, bool *eight_bit) {
|
||||||
int r = 1;
|
int r = 1;
|
||||||
|
|
||||||
assert(p);
|
assert(p);
|
||||||
|
@ -230,7 +230,7 @@ int cunescape_one(const char *p, size_t length, uint32_t *ret, bool *eight_bit)
|
||||||
|
|
||||||
int a[8];
|
int a[8];
|
||||||
unsigned i;
|
unsigned i;
|
||||||
uint32_t c;
|
char32_t c;
|
||||||
|
|
||||||
if (length != (size_t) -1 && length < 9)
|
if (length != (size_t) -1 && length < 9)
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
|
@ -267,7 +267,7 @@ int cunescape_one(const char *p, size_t length, uint32_t *ret, bool *eight_bit)
|
||||||
case '7': {
|
case '7': {
|
||||||
/* octal encoding */
|
/* octal encoding */
|
||||||
int a, b, c;
|
int a, b, c;
|
||||||
uint32_t m;
|
char32_t m;
|
||||||
|
|
||||||
if (length != (size_t) -1 && length < 3)
|
if (length != (size_t) -1 && length < 3)
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
|
@ -327,8 +327,8 @@ int cunescape_length_with_prefix(const char *s, size_t length, const char *prefi
|
||||||
|
|
||||||
for (f = s, t = r + pl; f < s + length; f++) {
|
for (f = s, t = r + pl; f < s + length; f++) {
|
||||||
size_t remaining;
|
size_t remaining;
|
||||||
uint32_t u;
|
|
||||||
bool eight_bit = false;
|
bool eight_bit = false;
|
||||||
|
char32_t u;
|
||||||
int k;
|
int k;
|
||||||
|
|
||||||
remaining = s + length - f;
|
remaining = s + length - f;
|
||||||
|
|
|
@ -25,8 +25,10 @@
|
||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
|
#include <uchar.h>
|
||||||
|
|
||||||
#include "string-util.h"
|
#include "string-util.h"
|
||||||
|
#include "missing.h"
|
||||||
|
|
||||||
/* What characters are special in the shell? */
|
/* What characters are special in the shell? */
|
||||||
/* must be escaped outside and inside double-quotes */
|
/* must be escaped outside and inside double-quotes */
|
||||||
|
@ -45,7 +47,7 @@ size_t cescape_char(char c, char *buf);
|
||||||
int cunescape(const char *s, UnescapeFlags flags, char **ret);
|
int cunescape(const char *s, UnescapeFlags flags, char **ret);
|
||||||
int cunescape_length(const char *s, size_t length, UnescapeFlags flags, char **ret);
|
int cunescape_length(const char *s, size_t length, UnescapeFlags flags, char **ret);
|
||||||
int cunescape_length_with_prefix(const char *s, size_t length, const char *prefix, UnescapeFlags flags, char **ret);
|
int cunescape_length_with_prefix(const char *s, size_t length, const char *prefix, UnescapeFlags flags, char **ret);
|
||||||
int cunescape_one(const char *p, size_t length, uint32_t *ret, bool *eight_bit);
|
int cunescape_one(const char *p, size_t length, char32_t *ret, bool *eight_bit);
|
||||||
|
|
||||||
char *xescape(const char *s, const char *bad);
|
char *xescape(const char *s, const char *bad);
|
||||||
|
|
||||||
|
|
|
@ -107,8 +107,8 @@ int extract_first_word(const char **p, char **ret, const char *separators, Extra
|
||||||
}
|
}
|
||||||
|
|
||||||
if (flags & EXTRACT_CUNESCAPE) {
|
if (flags & EXTRACT_CUNESCAPE) {
|
||||||
uint32_t u;
|
|
||||||
bool eight_bit = false;
|
bool eight_bit = false;
|
||||||
|
char32_t u;
|
||||||
|
|
||||||
r = cunescape_one(*p, (size_t) -1, &u, &eight_bit);
|
r = cunescape_one(*p, (size_t) -1, &u, &eight_bit);
|
||||||
if (r < 0) {
|
if (r < 0) {
|
||||||
|
|
|
@ -322,7 +322,7 @@ static int json_parse_string(const char **p, char **ret) {
|
||||||
else if (*c == 't')
|
else if (*c == 't')
|
||||||
ch = '\t';
|
ch = '\t';
|
||||||
else if (*c == 'u') {
|
else if (*c == 'u') {
|
||||||
uint16_t x;
|
char16_t x;
|
||||||
int r;
|
int r;
|
||||||
|
|
||||||
r = unhex_ucs2(c + 1, &x);
|
r = unhex_ucs2(c + 1, &x);
|
||||||
|
@ -335,11 +335,11 @@ static int json_parse_string(const char **p, char **ret) {
|
||||||
return -ENOMEM;
|
return -ENOMEM;
|
||||||
|
|
||||||
if (!utf16_is_surrogate(x))
|
if (!utf16_is_surrogate(x))
|
||||||
n += utf8_encode_unichar(s + n, x);
|
n += utf8_encode_unichar(s + n, (char32_t) x);
|
||||||
else if (utf16_is_trailing_surrogate(x))
|
else if (utf16_is_trailing_surrogate(x))
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
else {
|
else {
|
||||||
uint16_t y;
|
char16_t y;
|
||||||
|
|
||||||
if (c[0] != '\\' || c[1] != 'u')
|
if (c[0] != '\\' || c[1] != 'u')
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
|
|
|
@ -36,6 +36,7 @@
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <sys/resource.h>
|
#include <sys/resource.h>
|
||||||
#include <sys/syscall.h>
|
#include <sys/syscall.h>
|
||||||
|
#include <uchar.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
#ifdef HAVE_AUDIT
|
#ifdef HAVE_AUDIT
|
||||||
|
@ -1159,4 +1160,13 @@ static inline key_serial_t request_key(const char *type, const char *description
|
||||||
|
|
||||||
#ifndef IF_OPER_UP
|
#ifndef IF_OPER_UP
|
||||||
#define IF_OPER_UP 6
|
#define IF_OPER_UP 6
|
||||||
|
|
||||||
|
#ifndef HAVE_DECL_CHAR32_T
|
||||||
|
#define char32_t uint32_t
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifndef HAVE_DECL_CHAR16_T
|
||||||
|
#define char16_t uint16_t
|
||||||
|
#endif
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -450,6 +450,7 @@ char *ellipsize_mem(const char *s, size_t old_length, size_t new_length, unsigne
|
||||||
char *e;
|
char *e;
|
||||||
const char *i, *j;
|
const char *i, *j;
|
||||||
unsigned k, len, len2;
|
unsigned k, len, len2;
|
||||||
|
int r;
|
||||||
|
|
||||||
assert(s);
|
assert(s);
|
||||||
assert(percent <= 100);
|
assert(percent <= 100);
|
||||||
|
@ -469,10 +470,10 @@ char *ellipsize_mem(const char *s, size_t old_length, size_t new_length, unsigne
|
||||||
|
|
||||||
k = 0;
|
k = 0;
|
||||||
for (i = s; k < x && i < s + old_length; i = utf8_next_char(i)) {
|
for (i = s; k < x && i < s + old_length; i = utf8_next_char(i)) {
|
||||||
int c;
|
char32_t c;
|
||||||
|
|
||||||
c = utf8_encoded_to_unichar(i);
|
r = utf8_encoded_to_unichar(i, &c);
|
||||||
if (c < 0)
|
if (r < 0)
|
||||||
return NULL;
|
return NULL;
|
||||||
k += unichar_iswide(c) ? 2 : 1;
|
k += unichar_iswide(c) ? 2 : 1;
|
||||||
}
|
}
|
||||||
|
@ -481,11 +482,11 @@ char *ellipsize_mem(const char *s, size_t old_length, size_t new_length, unsigne
|
||||||
x ++;
|
x ++;
|
||||||
|
|
||||||
for (j = s + old_length; k < new_length && j > i; ) {
|
for (j = s + old_length; k < new_length && j > i; ) {
|
||||||
int c;
|
char32_t c;
|
||||||
|
|
||||||
j = utf8_prev_char(j);
|
j = utf8_prev_char(j);
|
||||||
c = utf8_encoded_to_unichar(j);
|
r = utf8_encoded_to_unichar(j, &c);
|
||||||
if (c < 0)
|
if (r < 0)
|
||||||
return NULL;
|
return NULL;
|
||||||
k += unichar_iswide(c) ? 2 : 1;
|
k += unichar_iswide(c) ? 2 : 1;
|
||||||
}
|
}
|
||||||
|
|
|
@ -53,7 +53,7 @@
|
||||||
#include "macro.h"
|
#include "macro.h"
|
||||||
#include "utf8.h"
|
#include "utf8.h"
|
||||||
|
|
||||||
bool unichar_is_valid(uint32_t ch) {
|
bool unichar_is_valid(char32_t ch) {
|
||||||
|
|
||||||
if (ch >= 0x110000) /* End of unicode space */
|
if (ch >= 0x110000) /* End of unicode space */
|
||||||
return false;
|
return false;
|
||||||
|
@ -67,7 +67,7 @@ bool unichar_is_valid(uint32_t ch) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool unichar_is_control(uint32_t ch) {
|
static bool unichar_is_control(char32_t ch) {
|
||||||
|
|
||||||
/*
|
/*
|
||||||
0 to ' '-1 is the C0 range.
|
0 to ' '-1 is the C0 range.
|
||||||
|
@ -103,8 +103,9 @@ static int utf8_encoded_expected_len(const char *str) {
|
||||||
}
|
}
|
||||||
|
|
||||||
/* decode one unicode char */
|
/* decode one unicode char */
|
||||||
int utf8_encoded_to_unichar(const char *str) {
|
int utf8_encoded_to_unichar(const char *str, char32_t *ret_unichar) {
|
||||||
int unichar, len, i;
|
char32_t unichar;
|
||||||
|
int len, i;
|
||||||
|
|
||||||
assert(str);
|
assert(str);
|
||||||
|
|
||||||
|
@ -112,34 +113,37 @@ int utf8_encoded_to_unichar(const char *str) {
|
||||||
|
|
||||||
switch (len) {
|
switch (len) {
|
||||||
case 1:
|
case 1:
|
||||||
return (int)str[0];
|
*ret_unichar = (char32_t)str[0];
|
||||||
|
return 0;
|
||||||
case 2:
|
case 2:
|
||||||
unichar = str[0] & 0x1f;
|
unichar = str[0] & 0x1f;
|
||||||
break;
|
break;
|
||||||
case 3:
|
case 3:
|
||||||
unichar = (int)str[0] & 0x0f;
|
unichar = (char32_t)str[0] & 0x0f;
|
||||||
break;
|
break;
|
||||||
case 4:
|
case 4:
|
||||||
unichar = (int)str[0] & 0x07;
|
unichar = (char32_t)str[0] & 0x07;
|
||||||
break;
|
break;
|
||||||
case 5:
|
case 5:
|
||||||
unichar = (int)str[0] & 0x03;
|
unichar = (char32_t)str[0] & 0x03;
|
||||||
break;
|
break;
|
||||||
case 6:
|
case 6:
|
||||||
unichar = (int)str[0] & 0x01;
|
unichar = (char32_t)str[0] & 0x01;
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
}
|
}
|
||||||
|
|
||||||
for (i = 1; i < len; i++) {
|
for (i = 1; i < len; i++) {
|
||||||
if (((int)str[i] & 0xc0) != 0x80)
|
if (((char32_t)str[i] & 0xc0) != 0x80)
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
unichar <<= 6;
|
unichar <<= 6;
|
||||||
unichar |= (int)str[i] & 0x3f;
|
unichar |= (char32_t)str[i] & 0x3f;
|
||||||
}
|
}
|
||||||
|
|
||||||
return unichar;
|
*ret_unichar = unichar;
|
||||||
|
|
||||||
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool utf8_is_printable_newline(const char* str, size_t length, bool newline) {
|
bool utf8_is_printable_newline(const char* str, size_t length, bool newline) {
|
||||||
|
@ -148,15 +152,16 @@ bool utf8_is_printable_newline(const char* str, size_t length, bool newline) {
|
||||||
assert(str);
|
assert(str);
|
||||||
|
|
||||||
for (p = str; length;) {
|
for (p = str; length;) {
|
||||||
int encoded_len, val;
|
int encoded_len, r;
|
||||||
|
char32_t val;
|
||||||
|
|
||||||
encoded_len = utf8_encoded_valid_unichar(p);
|
encoded_len = utf8_encoded_valid_unichar(p);
|
||||||
if (encoded_len < 0 ||
|
if (encoded_len < 0 ||
|
||||||
(size_t) encoded_len > length)
|
(size_t) encoded_len > length)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
val = utf8_encoded_to_unichar(p);
|
r = utf8_encoded_to_unichar(p, &val);
|
||||||
if (val < 0 ||
|
if (r < 0 ||
|
||||||
unichar_is_control(val) ||
|
unichar_is_control(val) ||
|
||||||
(!newline && val == '\n'))
|
(!newline && val == '\n'))
|
||||||
return false;
|
return false;
|
||||||
|
@ -276,7 +281,7 @@ char *ascii_is_valid(const char *str) {
|
||||||
* Returns: The length in bytes that the UTF-8 representation does or would
|
* Returns: The length in bytes that the UTF-8 representation does or would
|
||||||
* occupy.
|
* occupy.
|
||||||
*/
|
*/
|
||||||
size_t utf8_encode_unichar(char *out_utf8, uint32_t g) {
|
size_t utf8_encode_unichar(char *out_utf8, char32_t g) {
|
||||||
|
|
||||||
if (g < (1 << 7)) {
|
if (g < (1 << 7)) {
|
||||||
if (out_utf8)
|
if (out_utf8)
|
||||||
|
@ -320,7 +325,7 @@ char *utf16_to_utf8(const void *s, size_t length) {
|
||||||
t = r;
|
t = r;
|
||||||
|
|
||||||
while (f < (const uint8_t*) s + length) {
|
while (f < (const uint8_t*) s + length) {
|
||||||
uint16_t w1, w2;
|
char16_t w1, w2;
|
||||||
|
|
||||||
/* see RFC 2781 section 2.2 */
|
/* see RFC 2781 section 2.2 */
|
||||||
|
|
||||||
|
@ -354,7 +359,7 @@ char *utf16_to_utf8(const void *s, size_t length) {
|
||||||
}
|
}
|
||||||
|
|
||||||
/* expected size used to encode one unicode char */
|
/* expected size used to encode one unicode char */
|
||||||
static int utf8_unichar_to_encoded_len(int unichar) {
|
static int utf8_unichar_to_encoded_len(char32_t unichar) {
|
||||||
|
|
||||||
if (unichar < 0x80)
|
if (unichar < 0x80)
|
||||||
return 1;
|
return 1;
|
||||||
|
@ -372,7 +377,8 @@ static int utf8_unichar_to_encoded_len(int unichar) {
|
||||||
|
|
||||||
/* validate one encoded unicode char and return its length */
|
/* validate one encoded unicode char and return its length */
|
||||||
int utf8_encoded_valid_unichar(const char *str) {
|
int utf8_encoded_valid_unichar(const char *str) {
|
||||||
int len, unichar, i;
|
int len, i, r;
|
||||||
|
char32_t unichar;
|
||||||
|
|
||||||
assert(str);
|
assert(str);
|
||||||
|
|
||||||
|
@ -389,7 +395,9 @@ int utf8_encoded_valid_unichar(const char *str) {
|
||||||
if ((str[i] & 0x80) != 0x80)
|
if ((str[i] & 0x80) != 0x80)
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
|
|
||||||
unichar = utf8_encoded_to_unichar(str);
|
r = utf8_encoded_to_unichar(str, &unichar);
|
||||||
|
if (r < 0)
|
||||||
|
return r;
|
||||||
|
|
||||||
/* check if encoded length matches encoded value */
|
/* check if encoded length matches encoded value */
|
||||||
if (utf8_unichar_to_encoded_len(unichar) != len)
|
if (utf8_unichar_to_encoded_len(unichar) != len)
|
||||||
|
|
|
@ -24,12 +24,14 @@
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
|
#include <uchar.h>
|
||||||
|
|
||||||
#include "macro.h"
|
#include "macro.h"
|
||||||
|
#include "missing.h"
|
||||||
|
|
||||||
#define UTF8_REPLACEMENT_CHARACTER "\xef\xbf\xbd"
|
#define UTF8_REPLACEMENT_CHARACTER "\xef\xbf\xbd"
|
||||||
|
|
||||||
bool unichar_is_valid(uint32_t c);
|
bool unichar_is_valid(char32_t c);
|
||||||
|
|
||||||
const char *utf8_is_valid(const char *s) _pure_;
|
const char *utf8_is_valid(const char *s) _pure_;
|
||||||
char *ascii_is_valid(const char *s) _pure_;
|
char *ascii_is_valid(const char *s) _pure_;
|
||||||
|
@ -40,20 +42,20 @@ bool utf8_is_printable_newline(const char* str, size_t length, bool newline) _pu
|
||||||
char *utf8_escape_invalid(const char *s);
|
char *utf8_escape_invalid(const char *s);
|
||||||
char *utf8_escape_non_printable(const char *str);
|
char *utf8_escape_non_printable(const char *str);
|
||||||
|
|
||||||
size_t utf8_encode_unichar(char *out_utf8, uint32_t g);
|
size_t utf8_encode_unichar(char *out_utf8, char32_t g);
|
||||||
char *utf16_to_utf8(const void *s, size_t length);
|
char *utf16_to_utf8(const void *s, size_t length);
|
||||||
|
|
||||||
int utf8_encoded_valid_unichar(const char *str);
|
int utf8_encoded_valid_unichar(const char *str);
|
||||||
int utf8_encoded_to_unichar(const char *str);
|
int utf8_encoded_to_unichar(const char *str, char32_t *ret_unichar);
|
||||||
|
|
||||||
static inline bool utf16_is_surrogate(uint16_t c) {
|
static inline bool utf16_is_surrogate(char16_t c) {
|
||||||
return (0xd800 <= c && c <= 0xdfff);
|
return (0xd800 <= c && c <= 0xdfff);
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline bool utf16_is_trailing_surrogate(uint16_t c) {
|
static inline bool utf16_is_trailing_surrogate(char16_t c) {
|
||||||
return (0xdc00 <= c && c <= 0xdfff);
|
return (0xdc00 <= c && c <= 0xdfff);
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline uint32_t utf16_surrogate_pair_to_unichar(uint16_t lead, uint16_t trail) {
|
static inline char32_t utf16_surrogate_pair_to_unichar(char16_t lead, char16_t trail) {
|
||||||
return ((lead - 0xd800) << 10) + (trail - 0xdc00) + 0x10000;
|
return ((lead - 0xd800) << 10) + (trail - 0xdc00) + 0x10000;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue