-
Notifications
You must be signed in to change notification settings - Fork 0
/
lutf8proc.c
601 lines (539 loc) · 17.3 KB
/
lutf8proc.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
#include <stdlib.h>
#include <string.h>
#include <stdbool.h>
#include "compat-5.3.h"
#include "utf8proc.h"
// Use bit options in map and map_custom.
// #define USE_BIT_OPTIONS
#define LUA_SETTYPE(L, k, v, type) \
lua_push##type(L, v), lua_setfield(L, -2, k)
#define TO_UPPER(ch) \
(('a' <= (ch) && (ch) <= 'z') ? (ch) - 0x20 : (ch))
#define CODEPOINT_IN_RANGE(cp) (0 <= (cp) && (cp) <= 0x10FFFF)
#define ARRAY_LENGTH(arr) (sizeof (arr) / sizeof (arr)[0])
typedef struct category_desc {
const char code[3];
const char * const description;
} category_desc;
// PropertyValueAliases.txt
// The categories must be in the same order as in the enum in utf8proc.h!
static const category_desc category_names[] = {
{ "Cn", "Other, not assigned" },
{ "Lu", "Letter, uppercase" },
{ "Ll", "Letter, lowercase" },
{ "Lt", "Letter, titlecase" },
{ "Lm", "Letter, modifier" },
{ "Lo", "Letter, other" },
{ "Mn", "Mark, nonspacing" },
{ "Mc", "Mark, spacing combining" },
{ "Me", "Mark, enclosing" },
{ "Nd", "Number, decimal digit" },
{ "Nl", "Number, letter" },
{ "No", "Number, other" },
{ "Pc", "Punctuation, connector" },
{ "Pd", "Punctuation, dash" },
{ "Ps", "Punctuation, open" },
{ "Pe", "Punctuation, close" },
{ "Pi", "Punctuation, initial quote" },
{ "Pf", "Punctuation, final quote" },
{ "Po", "Punctuation, other" },
{ "Sm", "Symbol, math" },
{ "Sc", "Symbol, currency" },
{ "Sk", "Symbol, modifier" },
{ "So", "Symbol, other" },
{ "Zs", "Separator, space" },
{ "Zl", "Separator, line" },
{ "Zp", "Separator, paragraph" },
{ "Cc", "Other, control" },
{ "Cf", "Other, format" },
{ "Cs", "Other, surrogate" },
{ "Co", "Other, private use" },
};
#define CATEGORY_COUNT (ARRAY_LENGTH(category_names))
#ifdef USE_BIT_OPTIONS
typedef struct option {
const char * const name;
const int value;
} option;
#ifdef USE_UPPER_OPTION
#define OPT(lower_option, upper_option) \
{ #upper_option, UTF8PROC_##upper_option }
#else
#define OPT(lower_option, upper_option) \
{ #lower_option, UTF8PROC_##upper_option }
#endif // USE_UPPER_OPTION
static const option option_list[] = {
OPT(nullterm, NULLTERM ),
OPT(stable, STABLE ),
OPT(compat, COMPAT ),
OPT(compose, COMPOSE ),
OPT(decompose, DECOMPOSE),
OPT(ignore, IGNORE ),
OPT(rejectna, REJECTNA ),
OPT(nlf2lf, NLF2LF ), // Order required by show_options.
OPT(nlf2ls, NLF2LS ),
OPT(nlf2ps, NLF2PS ),
OPT(stripcc, STRIPCC ),
OPT(casefold, CASEFOLD ),
OPT(charbound, CHARBOUND),
OPT(lump, LUMP ),
OPT(stripmark, STRIPMARK),
};
#undef DEFINE_OPTION
#define OPTION_COUNT (ARRAY_LENGTH(option_list))
#endif // USE_BIT_OPTIONS
#define MAX_OPTION_BIT 13
#define MASK_N(n) (~((1 << ((n) + 1)) - 1))
// Only the 13 lowest bits are used by options.
#define CHECK_OPTIONS(o) (((o) & MASK_N(MAX_OPTION_BIT)) == 0)
// undefined at bottom
#ifndef USE_BIT_OPTIONS
#ifdef WINDOWS
#define strcasecmp _stricmp
#endif // WINDOWS
static int lua_case_insensitive_check_option (lua_State * L, int arg,
const char *def,
const char * const options[]) {
const char *name = (def)
? luaL_optstring(L, arg, def) : luaL_checkstring(L, arg);
for (size_t i = 0; options[i]; i++)
if (strcasecmp(options[i], name) == 0)
return i;
return luaL_argerror(L, arg,
lua_pushfstring(L, "invalid option '%s'", name));
}
#endif // not defined USE_BIT_OPTIONS
/*
utf8proc_ssize_t utf8proc_iterate
(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *codepoint_ref)
*/
/*
* If value at index is integer, return it; if string, return first codepoint.
* Error if beginning of string is invalid UTF-8.
*/
static utf8proc_int32_t get_codepoint (lua_State * L, int idx, int throw_err) {
utf8proc_int32_t codepoint;
switch (lua_type(L, idx)) {
case LUA_TSTRING: {
size_t str_len;
const char * str = luaL_checklstring(L, idx, &str_len);
if (str_len == 0) { /* Empty string is valid UTF-8. */
return 1;
}
/* Read single codepoint. Negative value is returned and written
to 'codepoint' variable in case of error. */
utf8proc_ssize_t err_code =
utf8proc_iterate((const utf8proc_uint8_t *) str,
(utf8proc_ssize_t) -1, &codepoint);
if (err_code < 0 && throw_err)
return luaL_error(L, utf8proc_errmsg(err_code));
else
break;
}
case LUA_TNUMBER:
codepoint = (utf8proc_int32_t) luaL_checkinteger(L, 1);
if (!CODEPOINT_IN_RANGE(codepoint)) {
if (throw_err)
return luaL_argerror(L, idx, "value out of range");
else
codepoint = -1;
}
break;
default: {
luaL_argerror(L, idx,
lua_pushfstring(L, "string or number expected, got %s",
luaL_typename(L, idx)));
}
}
return codepoint;
}
/*
const char *utf8proc_category_string(utf8proc_int32_t codepoint);
*/
/* Return the category name for a codepoint, or for the first UTF-8 character in a string. */
static int get_category (lua_State * L) {
utf8proc_int32_t codepoint = get_codepoint(L, 1, 1);
lua_pushstring(L, utf8proc_category_string(codepoint));
return 1;
}
/*
utf8proc_category_t utf8proc_category(utf8proc_int32_t codepoint);
*/
static int get_category_desc (lua_State * L) {
utf8proc_int32_t codepoint = get_codepoint(L, 1, 1);
lua_pushstring(L, category_names[utf8proc_category(codepoint)].description);
return 1;
}
/*
typedef utf8proc_int16_t utf8proc_propval_t;
typedef struct utf8proc_property_struct {
// Unicode category.
// see utf8proc_category_t
utf8proc_propval_t category;
utf8proc_propval_t combining_class;
// Bidirectional class.
// see utf8proc_bidi_class_t.
utf8proc_propval_t bidi_class;
// Decomposition type.
// see utf8proc_decomp_type_t.
utf8proc_propval_t decomp_type;
utf8proc_uint16_t decomp_seqindex;
utf8proc_uint16_t casefold_seqindex;
utf8proc_uint16_t uppercase_seqindex;
utf8proc_uint16_t lowercase_seqindex;
utf8proc_uint16_t titlecase_seqindex;
utf8proc_uint16_t comb_index;
unsigned bidi_mirrored:1;
unsigned comp_exclusion:1;
//Can this codepoint be ignored?
// Used by @ref utf8proc_decompose_char when @ref UTF8PROC_IGNORE is
// passed as an option.
unsigned ignorable:1;
unsigned control_boundary:1;
// The width of the codepoint.
unsigned charwidth:2;
unsigned pad:2;
// Boundclass.
// see utf8proc_boundclass_t.
unsigned boundclass:8;
} utf8proc_property_t;
*/
/* const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t codepoint) */
static int get_properties(lua_State * L) {
utf8proc_int32_t codepoint = get_codepoint(L, 1, 1);
const utf8proc_property_t * props = utf8proc_get_property(codepoint);
lua_newtable(L);
#define SET_PROP(prop, type) LUA_SETTYPE(L, #prop, props->prop, type)
SET_PROP(bidi_mirrored, boolean);
SET_PROP(control_boundary, boolean);
SET_PROP(comp_exclusion, boolean);
SET_PROP(category, integer);
SET_PROP(combining_class, integer);
SET_PROP(boundclass, integer);
#undef SET_PROP
return 1;
}
typedef enum {
NFD,
NFC,
NFKD,
NFKC
} normalization_t;
#define NORMALIZATION_CASE(normalization, str, new_str) \
case normalization: \
new_str = (char *) utf8proc_##normalization(str); \
break
/* utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str); */
static int normalize (lua_State * L, int idx, normalization_t mode) {
size_t str_len;
const utf8proc_uint8_t * str =
(const utf8proc_uint8_t *) luaL_checklstring(L, idx, &str_len);
if (str_len > 0) {
char * new_str;
switch (mode) {
NORMALIZATION_CASE(NFD, str, new_str);
NORMALIZATION_CASE(NFC, str, new_str);
NORMALIZATION_CASE(NFKD, str, new_str);
NORMALIZATION_CASE(NFKC, str, new_str);
default:
luaL_error(L, "invalid mode for normalization");
}
if (new_str == NULL)
luaL_error(L, "invalid UTF-8 sequence");
lua_pushstring(L, new_str);
free(new_str);
}
return 1;
}
#undef NORMALIZATION_CASE
/* Decompose a string (convert to NFD). */
static int decompose (lua_State * L) {
return normalize(L, 1, NFD);
}
/* Compose a string (convert to NFC). */
static int compose (lua_State * L) {
return normalize(L, 1, NFC);
}
static int compat_decompose (lua_State * L) {
return normalize(L, 1, NFKD);
}
static int compat_compose (lua_State * L) {
return normalize(L, 1, NFKC);
}
static int lua_normalize (lua_State * L) {
size_t mode_len;
const char * mode = luaL_checklstring(L, 2, &mode_len);
if ((mode_len == 3 || (mode_len == 4 && TO_UPPER(mode[2]) == 'K'))
&& TO_UPPER(mode[0]) == 'N' && TO_UPPER(mode[1]) == 'F') {
switch (TO_UPPER(mode[mode_len - 1])) {
case 'D':
return (mode_len == 4)
? normalize(L, 1, NFKD) : normalize(L, 1, NFD);
case 'C':
return (mode_len == 4)
? normalize(L, 1, NFKC) : normalize(L, 1, NFC);
}
}
return luaL_error(L, "normalization form '%s' not recognized; "
"valid forms 'NFD', 'NFC', 'NFKD', 'NFKC'",
mode);
}
/* Check validity of single codepoint (or first UTF-8 character in string). */
static int valid_codepoint (lua_State * L) {
utf8proc_int32_t codepoint = get_codepoint(L, 1, 0);
lua_pushboolean(L, codepoint >= 0 ? utf8proc_codepoint_valid(codepoint) : 0);
return 1;
}
/*
utf8proc_ssize_t utf8proc_iterate
(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *codepoint_ref)
*/
/* utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c); */
/* utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c); */
/* utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) */
// Use utf8proc_map_custom? Probably less efficient.
static int change_case (lua_State * L,
utf8proc_int32_t (* change_codepoint) (utf8proc_int32_t)) {
size_t str_len;
const char * str = luaL_checklstring(L, 1, &str_len);
utf8proc_uint8_t * result;
utf8proc_ssize_t new_len;
new_len = utf8proc_map_custom((const utf8proc_uint8_t *) str,
(utf8proc_ssize_t) str_len, &result, (utf8proc_option_t) 0,
(utf8proc_custom_func) change_codepoint, NULL);
if (new_len < 0)
return luaL_error(L, utf8proc_errmsg(new_len));
lua_pushlstring(L, (const char *) result, (size_t) new_len);
free(result);
return 1;
}
static int to_lower (lua_State * L) {
return change_case(L, utf8proc_tolower);
}
static int to_upper (lua_State * L) {
return change_case(L, utf8proc_toupper);
}
static int to_title (lua_State * L) {
return change_case(L, utf8proc_totitle);
}
static utf8proc_option_t get_options (lua_State * L, int arg, bool no_option_error) {
utf8proc_option_t options = 0;
#ifdef USE_BIT_OPTIONS
options = (utf8proc_option_t) luaL_checkinteger(L, arg); /* silently convert */
#else
static const char * option_strings[] = {
"nullterm",
"stable",
"compat",
"compose",
"decompose",
"ignore",
"rejectna",
"nlf2ls",
"nlf2ps",
"stripcc",
"casefold",
"charbound",
"lump",
"stripmark",
"nlf2lf",
NULL
};
// Make case-insensitive?
for (int top = lua_gettop(L); arg <= top; ++arg) {
utf8proc_option_t option = lua_case_insensitive_check_option(L, arg, NULL, option_strings);
if (option == ARRAY_LENGTH(option_strings) - 1)
option = UTF8PROC_NLF2LF;
else
option = 1 << option;
if ((option & options)
|| ((option & UTF8PROC_NLF2LF) && (options & UTF8PROC_NLF2LF)))
luaL_error(L, "conflicting options or option already set");
options |= option;
}
#endif // USE_BIT_OPTIONS
if (no_option_error && options == 0)
return luaL_error(L, "no options were selected");
else if (!CHECK_OPTIONS(options))
return luaL_error(L, "invalid options");
return options;
}
/*
utf8proc_ssize_t utf8proc_map(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
*/
// Convert to use luaL_checkoption.
static int lua_map (lua_State * L) {
utf8proc_option_t options;
size_t str_len;
const char * str = luaL_checklstring(L, 1, &str_len);
if (str_len == 0) { /* Return empty string. */
lua_settop(L, 1);
return 1;
}
options = get_options(L, 2, true);
utf8proc_uint8_t * result;
// Problematic if string is longer than maximum utf8proc_ssize_t.
utf8proc_ssize_t new_len =
utf8proc_map((const utf8proc_uint8_t *) str, (utf8proc_ssize_t) str_len,
&result, (utf8proc_option_t) options);
if (new_len < 0) /* error code */
return luaL_error(L, utf8proc_errmsg(new_len));
lua_pushlstring(L, (const char *) result, (size_t) new_len);
free(result);
return 1;
}
// typedef utf8proc_int32_t (*utf8proc_custom_func)(utf8proc_int32_t codepoint, void *data);
// Call function on top of stack on a codepoint.
// stack[3] must be function.
static utf8proc_int32_t custom_func (utf8proc_int32_t codepoint, void * data) {
lua_State * L = (lua_State *) data;
#ifdef USE_BIT_OPTIONS
lua_pushvalue(L, 3); // Push function.
#else
lua_pushvalue(L, 2); // Push function.
#endif // USE_BIT_OPTIONS
lua_pushinteger(L, (lua_Integer) codepoint); // Push codepoint.
lua_call(L, 1, 1); // func(codepoint)
utf8proc_int32_t result;
switch (lua_type(L, 4)) {
case LUA_TNUMBER: {
int isnum;
result = (utf8proc_int32_t) lua_tointegerx(L, 4, &isnum);
if (!isnum) {
luaL_error(L, "wrong return value (number has no integer representation)");
}
break;
}
case LUA_TNIL: case LUA_TNONE: /* Leave codepoint unchanged. */
result = codepoint; break;
default:
luaL_error(L, "wrong return value (integer expected, got %s)", luaL_typename(L, 4));
}
lua_settop(L, 3);
return result;
}
/*
utf8proc_ssize_t utf8proc_map_custom(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options,
utf8proc_custom_func custom_func, void *custom_data
);
*/
// Args: string, options (integer), function (receives codepoint, returns codepoint).
static int lua_map_custom (lua_State * L) {
utf8proc_option_t options;
size_t str_len;
const char * str = luaL_checklstring(L, 1, &str_len);
if (str_len == 0) { /* Return empty string. */
lua_settop(L, 1);
return 1;
}
#ifdef USE_BIT_OPTIONS
options = get_options(L, 2, false);
luaL_checktype(L, 3, LUA_TFUNCTION);
lua_settop(L, 3); // Ensure function is on top of stack.
#else
luaL_checktype(L, 2, LUA_TFUNCTION);
options = get_options(L, 3, false); // At the moment, options use varargs.
lua_settop(L, 2);
#endif // USE_BIT_OPTIONS
utf8proc_uint8_t * result;
// Problematic if string is longer than maximum utf8proc_ssize_t.
utf8proc_ssize_t new_len =
utf8proc_map_custom((const utf8proc_uint8_t *) str,
(utf8proc_ssize_t) str_len, &result,
(utf8proc_option_t) options,
(utf8proc_custom_func) custom_func, (void *) L);
if (new_len < 0) /* error code */
return luaL_error(L, utf8proc_errmsg(new_len));
lua_pushlstring(L, (const char *) result, (size_t) new_len);
free(result);
return 1;
}
#ifdef USE_BIT_OPTIONS
/* Return the names of the options set in the integer argument. */
static int interpret_options (lua_State * L) {
lua_Integer options = luaL_checkinteger(L, 1);
if (options == 0)
return 0;
const option * o;
int i = 0,
n = 0; // number of options
while (options && i < OPTION_COUNT) {
o = &option_list[i++];
if (options & o->value) {
lua_pushstring(L, o->name), ++n;
options &= ~o->value; // Remove option bits.
// Skip UTF8PROC_NLF2LS and UTF8PROC_NLF2PS.
if (o->value == UTF8PROC_NLF2LF)
i += 2;
}
}
if (options)
return luaL_error(L, "options invalid");
return n;
}
#endif // USE_BIT_OPTIONS
static int push_category_map (lua_State * L) {
lua_createtable(L, CATEGORY_COUNT, CATEGORY_COUNT * 2);
const category_desc * cat;
for (size_t i = 0; i < CATEGORY_COUNT; ++i) {
cat = &category_names[i];
LUA_SETTYPE(L, cat->code, cat->description, string); /* map[code] = description */
LUA_SETTYPE(L, cat->description, cat->code, string); /* map[description] = code */
lua_pushstring(L, cat->code);
lua_rawseti(L, -2, (lua_Integer) i + 1); /* map[index] = code */
}
return 1;
}
#ifdef USE_BIT_OPTIONS
static int push_options (lua_State * L) {
luaL_newlibtable(L, option_list);
for (int i = 0; i < OPTION_COUNT; ++i) {
lua_pushinteger(L, option_list[i].value);
lua_setfield(L, -2, option_list[i].name);
}
return 1;
}
#endif // USE_BIT_OPTIONS
static void push_utf8class (lua_State * L) {
lua_createtable(L, 255, 1);
for (size_t i = 0; i < ARRAY_LENGTH(utf8proc_utf8class); ++i) {
lua_pushinteger(L, utf8proc_utf8class[i]);
lua_rawseti(L, -2, (lua_Integer) i);
}
}
static luaL_Reg funcs[] = {
{ "cat", get_category },
{ "catdesc", get_category_desc },
{ "decomp", decompose },
{ "comp", compose },
{ "cdecomp", compat_decompose },
{ "ccomp", compat_compose },
{ "props", get_properties },
{ "valid", valid_codepoint },
{ "normalize", lua_normalize },
{ "lower", to_lower },
{ "upper", to_upper },
{ "title", to_title },
{ "map", lua_map },
{ "map_custom", lua_map_custom },
#ifdef USE_BIT_OPTIONS
{ "interpret_options", interpret_options },
#endif // USE_BIT_OPTIONS
{ NULL, NULL }
};
int luaopen_lutf8proc (lua_State * L) {
luaL_newlib(L, funcs);
lua_pushstring(L, utf8proc_version()), lua_setfield(L, -2, "_VERSION");
#define PUSH_SETFIELD(what, name) push_##what(L), lua_setfield(L, -2, name)
PUSH_SETFIELD(utf8class, "utf8class");
PUSH_SETFIELD(category_map, "categories");
#ifdef USE_BIT_OPTIONS
PUSH_SETFIELD(options, "options");
#endif
#undef PUSH_SETFIELD
return 1;
}
#undef MAX_OPTION_BIT
#undef MASK_N