1const std = @import("std");
2const windows1252 = @import("windows1252.zig");
3
4/// https://learn.microsoft.com/en-us/windows/win32/intl/code-page-identifiers
5pub const SupportedCodePage = enum(u16) {
6 windows1252 = 1252, // windows-1252 ANSI Latin 1; Western European (Windows)
7 utf8 = 65001, // utf-8 Unicode (UTF-8)
8
9 pub fn codepointAt(code_page: SupportedCodePage, index: usize, bytes: []const u8) ?Codepoint {
10 if (index >= bytes.len) return null;
11 switch (code_page) {
12 .windows1252 => {
13 // All byte values have a representation, so just convert the byte
14 return Codepoint{
15 .value = windows1252.toCodepoint(bytes[index]),
16 .byte_len = 1,
17 };
18 },
19 .utf8 => {
20 return Utf8.WellFormedDecoder.decode(bytes[index..]);
21 },
22 }
23 }
24};
25
26/// https://learn.microsoft.com/en-us/windows/win32/intl/code-page-identifiers
27pub const UnsupportedCodePage = enum(u16) {
28 ibm037 = 37, // IBM037 IBM EBCDIC US-Canada
29 ibm437 = 437, // IBM437 OEM United States
30 ibm500 = 500, // IBM500 IBM EBCDIC International
31 asmo708 = 708, // ASMO-708 Arabic (ASMO 708)
32 asmo449plus = 709, // Arabic (ASMO-449+, BCON V4)
33 transparent_arabic = 710, // Arabic - Transparent Arabic
34 dos720 = 720, // DOS-720 Arabic (Transparent ASMO); Arabic (DOS)
35 ibm737 = 737, // ibm737 OEM Greek (formerly 437G); Greek (DOS)
36 ibm775 = 775, // ibm775 OEM Baltic; Baltic (DOS)
37 ibm850 = 850, // ibm850 OEM Multilingual Latin 1; Western European (DOS)
38 ibm852 = 852, // ibm852 OEM Latin 2; Central European (DOS)
39 ibm855 = 855, // IBM855 OEM Cyrillic (primarily Russian)
40 ibm857 = 857, // ibm857 OEM Turkish; Turkish (DOS)
41 ibm00858 = 858, // IBM00858 OEM Multilingual Latin 1 + Euro symbol
42 ibm860 = 860, // IBM860 OEM Portuguese; Portuguese (DOS)
43 ibm861 = 861, // ibm861 OEM Icelandic; Icelandic (DOS)
44 dos862 = 862, // DOS-862 OEM Hebrew; Hebrew (DOS)
45 ibm863 = 863, // IBM863 OEM French Canadian; French Canadian (DOS)
46 ibm864 = 864, // IBM864 OEM Arabic; Arabic (864)
47 ibm865 = 865, // IBM865 OEM Nordic; Nordic (DOS)
48 cp866 = 866, // cp866 OEM Russian; Cyrillic (DOS)
49 ibm869 = 869, // ibm869 OEM Modern Greek; Greek, Modern (DOS)
50 ibm870 = 870, // IBM870 IBM EBCDIC Multilingual/ROECE (Latin 2); IBM EBCDIC Multilingual Latin 2
51 windows874 = 874, // windows-874 Thai (Windows)
52 cp875 = 875, // cp875 IBM EBCDIC Greek Modern
53 shift_jis = 932, // shift_jis ANSI/OEM Japanese; Japanese (Shift-JIS)
54 gb2312 = 936, // gb2312 ANSI/OEM Simplified Chinese (PRC, Singapore); Chinese Simplified (GB2312)
55 ks_c_5601_1987 = 949, // ks_c_5601-1987 ANSI/OEM Korean (Unified Hangul Code)
56 big5 = 950, // big5 ANSI/OEM Traditional Chinese (Taiwan; Hong Kong SAR, PRC); Chinese Traditional (Big5)
57 ibm1026 = 1026, // IBM1026 IBM EBCDIC Turkish (Latin 5)
58 ibm01047 = 1047, // IBM01047 IBM EBCDIC Latin 1/Open System
59 ibm01140 = 1140, // IBM01140 IBM EBCDIC US-Canada (037 + Euro symbol); IBM EBCDIC (US-Canada-Euro)
60 ibm01141 = 1141, // IBM01141 IBM EBCDIC Germany (20273 + Euro symbol); IBM EBCDIC (Germany-Euro)
61 ibm01142 = 1142, // IBM01142 IBM EBCDIC Denmark-Norway (20277 + Euro symbol); IBM EBCDIC (Denmark-Norway-Euro)
62 ibm01143 = 1143, // IBM01143 IBM EBCDIC Finland-Sweden (20278 + Euro symbol); IBM EBCDIC (Finland-Sweden-Euro)
63 ibm01144 = 1144, // IBM01144 IBM EBCDIC Italy (20280 + Euro symbol); IBM EBCDIC (Italy-Euro)
64 ibm01145 = 1145, // IBM01145 IBM EBCDIC Latin America-Spain (20284 + Euro symbol); IBM EBCDIC (Spain-Euro)
65 ibm01146 = 1146, // IBM01146 IBM EBCDIC United Kingdom (20285 + Euro symbol); IBM EBCDIC (UK-Euro)
66 ibm01147 = 1147, // IBM01147 IBM EBCDIC France (20297 + Euro symbol); IBM EBCDIC (France-Euro)
67 ibm01148 = 1148, // IBM01148 IBM EBCDIC International (500 + Euro symbol); IBM EBCDIC (International-Euro)
68 ibm01149 = 1149, // IBM01149 IBM EBCDIC Icelandic (20871 + Euro symbol); IBM EBCDIC (Icelandic-Euro)
69 utf16 = 1200, // utf-16 Unicode UTF-16, little endian byte order (BMP of ISO 10646); available only to managed applications
70 utf16_fffe = 1201, // unicodeFFFE Unicode UTF-16, big endian byte order; available only to managed applications
71 windows1250 = 1250, // windows-1250 ANSI Central European; Central European (Windows)
72 windows1251 = 1251, // windows-1251 ANSI Cyrillic; Cyrillic (Windows)
73 windows1253 = 1253, // windows-1253 ANSI Greek; Greek (Windows)
74 windows1254 = 1254, // windows-1254 ANSI Turkish; Turkish (Windows)
75 windows1255 = 1255, // windows-1255 ANSI Hebrew; Hebrew (Windows)
76 windows1256 = 1256, // windows-1256 ANSI Arabic; Arabic (Windows)
77 windows1257 = 1257, // windows-1257 ANSI Baltic; Baltic (Windows)
78 windows1258 = 1258, // windows-1258 ANSI/OEM Vietnamese; Vietnamese (Windows)
79 johab = 1361, // Johab Korean (Johab)
80 macintosh = 10000, // macintosh MAC Roman; Western European (Mac)
81 x_mac_japanese = 10001, // x-mac-japanese Japanese (Mac)
82 x_mac_chinesetrad = 10002, // x-mac-chinesetrad MAC Traditional Chinese (Big5); Chinese Traditional (Mac)
83 x_mac_korean = 10003, // x-mac-korean Korean (Mac)
84 x_mac_arabic = 10004, // x-mac-arabic Arabic (Mac)
85 x_mac_hebrew = 10005, // x-mac-hebrew Hebrew (Mac)
86 x_mac_greek = 10006, // x-mac-greek Greek (Mac)
87 x_mac_cyrillic = 10007, // x-mac-cyrillic Cyrillic (Mac)
88 x_mac_chinesesimp = 10008, // x-mac-chinesesimp MAC Simplified Chinese (GB 2312); Chinese Simplified (Mac)
89 x_mac_romanian = 10010, // x-mac-romanian Romanian (Mac)
90 x_mac_ukranian = 10017, // x-mac-ukrainian Ukrainian (Mac)
91 x_mac_thai = 10021, // x-mac-thai Thai (Mac)
92 x_mac_ce = 10029, // x-mac-ce MAC Latin 2; Central European (Mac)
93 x_mac_icelandic = 10079, // x-mac-icelandic Icelandic (Mac)
94 x_mac_turkish = 10081, // x-mac-turkish Turkish (Mac)
95 x_mac_croatian = 10082, // x-mac-croatian Croatian (Mac)
96 utf32 = 12000, // utf-32 Unicode UTF-32, little endian byte order; available only to managed applications
97 utf32_be = 12001, // utf-32BE Unicode UTF-32, big endian byte order; available only to managed applications
98 x_chinese_cns = 20000, // x-Chinese_CNS CNS Taiwan; Chinese Traditional (CNS)
99 x_cp20001 = 20001, // x-cp20001 TCA Taiwan
100 x_chinese_eten = 20002, // x_Chinese-Eten Eten Taiwan; Chinese Traditional (Eten)
101 x_cp20003 = 20003, // x-cp20003 IBM5550 Taiwan
102 x_cp20004 = 20004, // x-cp20004 TeleText Taiwan
103 x_cp20005 = 20005, // x-cp20005 Wang Taiwan
104 x_ia5 = 20105, // x-IA5 IA5 (IRV International Alphabet No. 5, 7-bit); Western European (IA5)
105 x_ia5_german = 20106, // x-IA5-German IA5 German (7-bit)
106 x_ia5_swedish = 20107, // x-IA5-Swedish IA5 Swedish (7-bit)
107 x_ia5_norwegian = 20108, // x-IA5-Norwegian IA5 Norwegian (7-bit)
108 us_ascii = 20127, // us-ascii US-ASCII (7-bit)
109 x_cp20261 = 20261, // x-cp20261 T.61
110 x_cp20269 = 20269, // x-cp20269 ISO 6937 Non-Spacing Accent
111 ibm273 = 20273, // IBM273 IBM EBCDIC Germany
112 ibm277 = 20277, // IBM277 IBM EBCDIC Denmark-Norway
113 ibm278 = 20278, // IBM278 IBM EBCDIC Finland-Sweden
114 ibm280 = 20280, // IBM280 IBM EBCDIC Italy
115 ibm284 = 20284, // IBM284 IBM EBCDIC Latin America-Spain
116 ibm285 = 20285, // IBM285 IBM EBCDIC United Kingdom
117 ibm290 = 20290, // IBM290 IBM EBCDIC Japanese Katakana Extended
118 ibm297 = 20297, // IBM297 IBM EBCDIC France
119 ibm420 = 20420, // IBM420 IBM EBCDIC Arabic
120 ibm423 = 20423, // IBM423 IBM EBCDIC Greek
121 ibm424 = 20424, // IBM424 IBM EBCDIC Hebrew
122 x_ebcdic_korean_extended = 20833, // x-EBCDIC-KoreanExtended IBM EBCDIC Korean Extended
123 ibm_thai = 20838, // IBM-Thai IBM EBCDIC Thai
124 koi8_r = 20866, // koi8-r Russian (KOI8-R); Cyrillic (KOI8-R)
125 ibm871 = 20871, // IBM871 IBM EBCDIC Icelandic
126 ibm880 = 20880, // IBM880 IBM EBCDIC Cyrillic Russian
127 ibm905 = 20905, // IBM905 IBM EBCDIC Turkish
128 ibm00924 = 20924, // IBM00924 IBM EBCDIC Latin 1/Open System (1047 + Euro symbol)
129 euc_jp_jis = 20932, // EUC-JP Japanese (JIS 0208-1990 and 0212-1990)
130 x_cp20936 = 20936, // x-cp20936 Simplified Chinese (GB2312); Chinese Simplified (GB2312-80)
131 x_cp20949 = 20949, // x-cp20949 Korean Wansung
132 cp1025 = 21025, // cp1025 IBM EBCDIC Cyrillic Serbian-Bulgarian
133 // = 21027, // (deprecated)
134 koi8_u = 21866, // koi8-u Ukrainian (KOI8-U); Cyrillic (KOI8-U)
135 iso8859_1 = 28591, // iso-8859-1 ISO 8859-1 Latin 1; Western European (ISO)
136 iso8859_2 = 28592, // iso-8859-2 ISO 8859-2 Central European; Central European (ISO)
137 iso8859_3 = 28593, // iso-8859-3 ISO 8859-3 Latin 3
138 iso8859_4 = 28594, // iso-8859-4 ISO 8859-4 Baltic
139 iso8859_5 = 28595, // iso-8859-5 ISO 8859-5 Cyrillic
140 iso8859_6 = 28596, // iso-8859-6 ISO 8859-6 Arabic
141 iso8859_7 = 28597, // iso-8859-7 ISO 8859-7 Greek
142 iso8859_8 = 28598, // iso-8859-8 ISO 8859-8 Hebrew; Hebrew (ISO-Visual)
143 iso8859_9 = 28599, // iso-8859-9 ISO 8859-9 Turkish
144 iso8859_13 = 28603, // iso-8859-13 ISO 8859-13 Estonian
145 iso8859_15 = 28605, // iso-8859-15 ISO 8859-15 Latin 9
146 x_europa = 29001, // x-Europa Europa 3
147 is8859_8_i = 38598, // iso-8859-8-i ISO 8859-8 Hebrew; Hebrew (ISO-Logical)
148 iso2022_jp = 50220, // iso-2022-jp ISO 2022 Japanese with no halfwidth Katakana; Japanese (JIS)
149 cs_iso2022_jp = 50221, // csISO2022JP ISO 2022 Japanese with halfwidth Katakana; Japanese (JIS-Allow 1 byte Kana)
150 iso2022_jp_jis_x = 50222, // iso-2022-jp ISO 2022 Japanese JIS X 0201-1989; Japanese (JIS-Allow 1 byte Kana - SO/SI)
151 iso2022_kr = 50225, // iso-2022-kr ISO 2022 Korean
152 x_cp50227 = 50227, // x-cp50227 ISO 2022 Simplified Chinese; Chinese Simplified (ISO 2022)
153 iso2022_chinesetrad = 50229, // ISO 2022 Traditional Chinese
154 ebcdic_jp_katakana_extended = 50930, // EBCDIC Japanese (Katakana) Extended
155 ebcdic_us_ca_jp = 50931, // EBCDIC US-Canada and Japanese
156 ebcdic_kr_extended = 50933, // EBCDIC Korean Extended and Korean
157 ebcdic_chinesesimp_extended = 50935, // EBCDIC Simplified Chinese Extended and Simplified Chinese
158 ebcdic_chinesesimp = 50936, // EBCDIC Simplified Chinese
159 ebcdic_us_ca_chinesetrad = 50937, // EBCDIC US-Canada and Traditional Chinese
160 ebcdic_jp_latin_extended = 50939, // EBCDIC Japanese (Latin) Extended and Japanese
161 euc_jp = 51932, // euc-jp EUC Japanese
162 euc_cn = 51936, // EUC-CN EUC Simplified Chinese; Chinese Simplified (EUC)
163 euc_kr = 51949, // euc-kr EUC Korean
164 euc_chinesetrad = 51950, // EUC Traditional Chinese
165 hz_gb2312 = 52936, // hz-gb-2312 HZ-GB2312 Simplified Chinese; Chinese Simplified (HZ)
166 gb18030 = 54936, // GB18030 Windows XP and later: GB18030 Simplified Chinese (4 byte); Chinese Simplified (GB18030)
167 x_iscii_de = 57002, // x-iscii-de ISCII Devanagari
168 x_iscii_be = 57003, // x-iscii-be ISCII Bangla
169 x_iscii_ta = 57004, // x-iscii-ta ISCII Tamil
170 x_iscii_te = 57005, // x-iscii-te ISCII Telugu
171 x_iscii_as = 57006, // x-iscii-as ISCII Assamese
172 x_iscii_or = 57007, // x-iscii-or ISCII Odia
173 x_iscii_ka = 57008, // x-iscii-ka ISCII Kannada
174 x_iscii_ma = 57009, // x-iscii-ma ISCII Malayalam
175 x_iscii_gu = 57010, // x-iscii-gu ISCII Gujarati
176 x_iscii_pa = 57011, // x-iscii-pa ISCII Punjabi
177 utf7 = 65000, // utf-7 Unicode (UTF-7)
178};
179
180pub const CodePage = blk: {
181 const field_names = @typeInfo(SupportedCodePage).@"enum".field_names ++ @typeInfo(UnsupportedCodePage).@"enum".field_names;
182 const field_values = @typeInfo(SupportedCodePage).@"enum".field_values ++ @typeInfo(UnsupportedCodePage).@"enum".field_values;
183 var cp_field_names: [field_names.len][]const u8 = undefined;
184 var cp_field_values: [field_names.len]u16 = undefined;
185 for (field_names, field_values, &cp_field_names, &cp_field_values) |field_name, field_value, *name, *val| {
186 name.* = field_name;
187 val.* = field_value;
188 }
189 break :blk @Enum(u16, .exhaustive, &cp_field_names, &cp_field_values);
190};
191
192pub fn isSupported(code_page: CodePage) bool {
193 inline for (@typeInfo(SupportedCodePage).@"enum".field_names) |field_name| {
194 if (@backingInt(code_page) == @backingInt(@field(SupportedCodePage, field_name))) {
195 return true;
196 }
197 }
198 return false;
199}
200
201pub fn getByIdentifier(identifier: u16) !CodePage {
202 // There's probably a more efficient way to do this (e.g. ComptimeHashMap?) but
203 // this should be fine, especially since this function likely won't be called much.
204 const info = @typeInfo(CodePage).@"enum";
205 inline for (info.field_names, info.field_values) |field_name, field_value| {
206 if (identifier == field_value) {
207 return @field(CodePage, field_name);
208 }
209 }
210 return error.InvalidCodePage;
211}
212
213pub fn getByIdentifierEnsureSupported(identifier: u16) !SupportedCodePage {
214 const code_page = try getByIdentifier(identifier);
215 return if (isSupported(code_page))
216 @fromBackingInt(@intCast(@backingInt(code_page)))
217 else
218 error.UnsupportedCodePage;
219}
220
221pub const Utf8 = struct {
222 /// Implements decoding with rejection of ill-formed UTF-8 sequences based on section
223 /// D92 of Chapter 3 of the Unicode standard (Table 3-7 specifically).
224 ///
225 /// Note: This does not match "U+FFFD Substitution of Maximal Subparts", but instead
226 /// matches the behavior of the Windows RC compiler.
227 pub const WellFormedDecoder = struct {
228 /// Like std.unicode.utf8ByteSequenceLength, but:
229 /// - Rejects non-well-formed first bytes, i.e. C0-C1, F5-FF
230 /// - Returns an optional value instead of an error union
231 pub fn sequenceLength(first_byte: u8) ?u3 {
232 return switch (first_byte) {
233 0x00...0x7F => 1,
234 0xC2...0xDF => 2,
235 0xE0...0xEF => 3,
236 0xF0...0xF4 => 4,
237 else => null,
238 };
239 }
240
241 fn isContinuationByte(byte: u8) bool {
242 return switch (byte) {
243 0x80...0xBF => true,
244 else => false,
245 };
246 }
247
248 pub fn decode(bytes: []const u8) Codepoint {
249 std.debug.assert(bytes.len > 0);
250 const first_byte = bytes[0];
251 const expected_len = sequenceLength(first_byte) orelse {
252 return .{ .value = Codepoint.invalid, .byte_len = 1 };
253 };
254 if (expected_len == 1) return .{ .value = first_byte, .byte_len = 1 };
255
256 var value: u21 = first_byte & 0b00011111;
257 var byte_index: u8 = 1;
258 while (byte_index < @min(bytes.len, expected_len)) : (byte_index += 1) {
259 const byte = bytes[byte_index];
260 // See Table 3-7 of D92 in Chapter 3 of the Unicode Standard
261 const valid: bool = switch (byte_index) {
262 1 => switch (first_byte) {
263 0xE0 => switch (byte) {
264 0xA0...0xBF => true,
265 else => false,
266 },
267 0xED => switch (byte) {
268 0x80...0x9F => true,
269 else => false,
270 },
271 0xF0 => switch (byte) {
272 0x90...0xBF => true,
273 else => false,
274 },
275 0xF4 => switch (byte) {
276 0x80...0x8F => true,
277 else => false,
278 },
279 else => switch (byte) {
280 0x80...0xBF => true,
281 else => false,
282 },
283 },
284 else => switch (byte) {
285 0x80...0xBF => true,
286 else => false,
287 },
288 };
289
290 if (!valid) {
291 var len = byte_index;
292 // Only include the byte in the invalid sequence if it's in the range
293 // of a continuation byte. All other values should not be included in the
294 // invalid sequence.
295 if (isContinuationByte(byte)) len += 1;
296 return .{ .value = Codepoint.invalid, .byte_len = len };
297 }
298
299 value <<= 6;
300 value |= byte & 0b00111111;
301 }
302 if (byte_index != expected_len) {
303 return .{ .value = Codepoint.invalid, .byte_len = byte_index };
304 }
305 return .{ .value = value, .byte_len = expected_len };
306 }
307 };
308};
309
310test "Utf8.WellFormedDecoder" {
311 const invalid_utf8 = "\xF0\x80";
312 const decoded = Utf8.WellFormedDecoder.decode(invalid_utf8);
313 try std.testing.expectEqual(Codepoint.invalid, decoded.value);
314 try std.testing.expectEqual(@as(usize, 2), decoded.byte_len);
315}
316
317test "codepointAt invalid utf8" {
318 {
319 const invalid_utf8 = "\xf0\xf0\x80\x80\x80";
320 try std.testing.expectEqual(Codepoint{
321 .value = Codepoint.invalid,
322 .byte_len = 1,
323 }, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
324 try std.testing.expectEqual(Codepoint{
325 .value = Codepoint.invalid,
326 .byte_len = 2,
327 }, SupportedCodePage.utf8.codepointAt(1, invalid_utf8).?);
328 try std.testing.expectEqual(Codepoint{
329 .value = Codepoint.invalid,
330 .byte_len = 1,
331 }, SupportedCodePage.utf8.codepointAt(3, invalid_utf8).?);
332 try std.testing.expectEqual(Codepoint{
333 .value = Codepoint.invalid,
334 .byte_len = 1,
335 }, SupportedCodePage.utf8.codepointAt(4, invalid_utf8).?);
336 try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(5, invalid_utf8));
337 }
338
339 {
340 const invalid_utf8 = "\xE1\xA0\xC0";
341 try std.testing.expectEqual(Codepoint{
342 .value = Codepoint.invalid,
343 .byte_len = 2,
344 }, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
345 try std.testing.expectEqual(Codepoint{
346 .value = Codepoint.invalid,
347 .byte_len = 1,
348 }, SupportedCodePage.utf8.codepointAt(2, invalid_utf8).?);
349 try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(3, invalid_utf8));
350 }
351
352 {
353 const invalid_utf8 = "\xD2";
354 try std.testing.expectEqual(Codepoint{
355 .value = Codepoint.invalid,
356 .byte_len = 1,
357 }, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
358 try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(1, invalid_utf8));
359 }
360
361 {
362 const invalid_utf8 = "\xE1\xA0";
363 try std.testing.expectEqual(Codepoint{
364 .value = Codepoint.invalid,
365 .byte_len = 2,
366 }, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
367 try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(2, invalid_utf8));
368 }
369
370 {
371 const invalid_utf8 = "\xC5\xFF";
372 try std.testing.expectEqual(Codepoint{
373 .value = Codepoint.invalid,
374 .byte_len = 1,
375 }, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
376 try std.testing.expectEqual(Codepoint{
377 .value = Codepoint.invalid,
378 .byte_len = 1,
379 }, SupportedCodePage.utf8.codepointAt(1, invalid_utf8).?);
380 try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(2, invalid_utf8));
381 }
382
383 {
384 // encoded high surrogate
385 const invalid_utf8 = "\xED\xA0\xBD";
386 try std.testing.expectEqual(Codepoint{
387 .value = Codepoint.invalid,
388 .byte_len = 2,
389 }, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
390 try std.testing.expectEqual(Codepoint{
391 .value = Codepoint.invalid,
392 .byte_len = 1,
393 }, SupportedCodePage.utf8.codepointAt(2, invalid_utf8).?);
394 }
395}
396
397test "codepointAt utf8 encoded" {
398 const utf8_encoded = "²";
399
400 // with code page utf8
401 try std.testing.expectEqual(Codepoint{
402 .value = '²',
403 .byte_len = 2,
404 }, SupportedCodePage.utf8.codepointAt(0, utf8_encoded).?);
405 try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(2, utf8_encoded));
406
407 // with code page windows1252
408 try std.testing.expectEqual(Codepoint{
409 .value = '\xC2',
410 .byte_len = 1,
411 }, SupportedCodePage.windows1252.codepointAt(0, utf8_encoded).?);
412 try std.testing.expectEqual(Codepoint{
413 .value = '\xB2',
414 .byte_len = 1,
415 }, SupportedCodePage.windows1252.codepointAt(1, utf8_encoded).?);
416 try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.windows1252.codepointAt(2, utf8_encoded));
417}
418
419test "codepointAt windows1252 encoded" {
420 const windows1252_encoded = "\xB2";
421
422 // with code page utf8
423 try std.testing.expectEqual(Codepoint{
424 .value = Codepoint.invalid,
425 .byte_len = 1,
426 }, SupportedCodePage.utf8.codepointAt(0, windows1252_encoded).?);
427 try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(2, windows1252_encoded));
428
429 // with code page windows1252
430 try std.testing.expectEqual(Codepoint{
431 .value = '\xB2',
432 .byte_len = 1,
433 }, SupportedCodePage.windows1252.codepointAt(0, windows1252_encoded).?);
434 try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.windows1252.codepointAt(1, windows1252_encoded));
435}
436
437pub const Codepoint = struct {
438 value: u21,
439 byte_len: usize,
440
441 pub const invalid: u21 = std.math.maxInt(u21);
442};