authorgravatar for tgschultz@gmail.comtgschultz <tgschultz@gmail.com> 2020-06-11 18:42:56+00:00
committergravatar for tgschultz@gmail.comtgschultz <tgschultz@gmail.com> 2020-06-16 16:20:59+00:00
log00ec81b0dc41e113624152cdb268d06f73e6f88c
tree3ed3a04950071666925397249077922af97ba3b3
parent928d3ee9ea7f1a7eb17b1ec3a8ec36559e1e9968

Overhauled leb128:

handles integers < 8 bits incorrect overflow bugs fixed simplified *mem implementations added wrte* functions added thurough write/read testing

1 files changed, 34 insertions(+), 45 deletions(-)

lib/std/debug/leb128.zig+34-45
...@@ -1,8 +1,11 @@...@@ -1,8 +1,11 @@
1const std = @import("std");1const std = @import("std");
2const testing = std.testing;2const testing = std.testing;
33
4///Read a single unsigned LEB128 value from the given reader as type T,4//@TODO: you can take *slice and alter slice.ptr
5/// or error.Overflow if the value cannot fit.5// make sign bits check more efficient
6// add wrapper readLEB128 and write LEB128 that infer from type?
7// or use assertions?
8
6pub fn readULEB128(comptime T: type, reader: var) !T {9pub fn readULEB128(comptime T: type, reader: var) !T {
7 const U = if (T.bit_count < 8) u8 else T;10 const U = if (T.bit_count < 8) u8 else T;
8 const ShiftT = std.math.Log2Int(U);11 const ShiftT = std.math.Log2Int(U);
...@@ -25,14 +28,11 @@ pub fn readULEB128(comptime T: type, reader: var) !T {...@@ -25,14 +28,11 @@ pub fn readULEB128(comptime T: type, reader: var) !T {
25 }28 }
2629
27 //only applies in the case that we extended to u830 //only applies in the case that we extended to u8
28 if (U != T) {31 if (value > std.math.maxInt(T)) return error.Overflow;
29 if (value > std.math.maxInt(T)) return error.Overflow;
30 }
3132
32 return @truncate(T, value);33 return @truncate(T, value);
33}34}
3435
35///Write a single unsigned integer as unsigned LEB128 to the given writer.
36pub fn writeULEB128(writer: var, uint_value: var) !void {36pub fn writeULEB128(writer: var, uint_value: var) !void {
37 const T = @TypeOf(uint_value);37 const T = @TypeOf(uint_value);
38 const U = if (T.bit_count < 8) u8 else T;38 const U = if (T.bit_count < 8) u8 else T;
...@@ -50,27 +50,22 @@ pub fn writeULEB128(writer: var, uint_value: var) !void {...@@ -50,27 +50,22 @@ pub fn writeULEB128(writer: var, uint_value: var) !void {
50 }50 }
51}51}
5252
53///Read a single unsinged integer from the given memory as type T.53pub fn readULEB128Mem(comptime T: type, ptr: *[*]const u8) !T {
54/// The provided slice reference will be updated to point to the byte after the last byte read.54 const max_group = (T.bit_count + 6) / 7;
55pub fn readULEB128Mem(comptime T: type, ptr: *[]const u8) !T {55 var buf = std.io.fixedBufferStream(ptr.*[0 .. max_group + 1]);
56 var buf = std.io.fixedBufferStream(ptr.*);
57 const value = try readULEB128(T, buf.reader());56 const value = try readULEB128(T, buf.reader());
58 ptr.*.ptr += buf.pos;57 ptr.* += @intCast(usize, try buf.getPos());
59 return value;58 return value;
60}59}
6160
62///Write a single unsigned LEB128 integer to the given memory as unsigned LEB128,
63/// returning the number of bytes written.
64pub fn writeULEB128Mem(ptr: []u8, uint_value: var) !usize {61pub fn writeULEB128Mem(ptr: []u8, uint_value: var) !usize {
65 const T = @TypeOf(uint_value);62 const T = @TypeOf(uint_value);
66 const max_group = (T.bit_count + 6) / 7;63 const max_group = (T.bit_count + 6) / 7;
67 var buf = std.io.fixedBufferStream(ptr);64 var buf = std.io.fixedBufferStream(ptr);
68 try writeULEB128(buf.writer(), uint_value);65 try writeULEB128(buf.writer(), uint_value);
69 return buf.pos;66 return try buf.getPos();
70}67}
7168
72///Read a single signed LEB128 value from the given reader as type T,
73/// or error.Overflow if the value cannot fit.
74pub fn readILEB128(comptime T: type, reader: var) !T {69pub fn readILEB128(comptime T: type, reader: var) !T {
75 const S = if (T.bit_count < 8) i8 else T;70 const S = if (T.bit_count < 8) i8 else T;
76 const U = std.meta.Int(false, S.bit_count);71 const U = std.meta.Int(false, S.bit_count);
...@@ -85,24 +80,23 @@ pub fn readILEB128(comptime T: type, reader: var) !T {...@@ -85,24 +80,23 @@ pub fn readILEB128(comptime T: type, reader: var) !T {
85 const byte = try reader.readByte();80 const byte = try reader.readByte();
86 var temp = @as(U, byte & 0x7f);81 var temp = @as(U, byte & 0x7f);
8782
88 const shift = group * 7;83 if (@shlWithOverflow(U, temp, group * 7, &temp)) {
89 if (@shlWithOverflow(U, temp, shift, &temp)) {
90 //Overflow is ok so long as the sign bit is set and this is the last byte84 //Overflow is ok so long as the sign bit is set and this is the last byte
91 if (byte & 0x80 != 0) return error.Overflow;85 if (byte & 0x80 != 0) return error.Overflow;
92 if (@bitCast(S, temp) >= 0) return error.Overflow;86 if (@bitCast(S, temp) >= 0) return error.Overflow;
9387
94 //and all the overflowed bits are 188 //and all the overflowed bits are 1
95 const remaining_shift = @intCast(u3, U.bit_count - @as(u16, shift));89 const check_bits_shift = @intCast(u3, U.bit_count - @as(u16, group * 7));
96 const remaining_bits = @bitCast(i8, byte | 0x80) >> remaining_shift;90 const check_bits_remaining = 7 - check_bits_shift;
97 if (remaining_bits != -1) return error.Overflow;91 const check_bits = byte >> check_bits_shift;
92 const num_consecutive_ones = @ctz(u8, ~check_bits);
93 if (num_consecutive_ones < check_bits_remaining) return error.Overflow;
98 }94 }
9995
100 value |= temp;96 value |= temp;
101 if (byte & 0x80 == 0) {97 if (byte & 0x80 == 0) {
102 const needs_sign_ext = group + 1 < max_group;98 if (byte & 0x40 != 0 and group + 1 < max_group) {
103 if (byte & 0x40 != 0 and needs_sign_ext) {99 value |= @bitCast(U, @as(S, -1)) << ((group + 1) * 7);
104 const ones = @as(S, -1);
105 value |= @bitCast(U, ones) << (shift + 7);
106 }100 }
107 break;101 break;
108 }102 }
...@@ -110,16 +104,12 @@ pub fn readILEB128(comptime T: type, reader: var) !T {...@@ -110,16 +104,12 @@ pub fn readILEB128(comptime T: type, reader: var) !T {
110 return error.Overflow;104 return error.Overflow;
111 }105 }
112106
113 const result = @bitCast(S, value);
114 //Only applies if we extended to i8107 //Only applies if we extended to i8
115 if (S != T) {108 if (@bitCast(S, value) > std.math.maxInt(T) or @bitCast(S, value) < std.math.minInt(T)) return error.Overflow;
116 if (result > std.math.maxInt(T) or result < std.math.minInt(T)) return error.Overflow;
117 }
118109
119 return @truncate(T, result);110 return @truncate(T, @bitCast(S, value));
120}111}
121112
122///Write a single signed integer as signed LEB128 to the given writer.
123pub fn writeILEB128(writer: var, int_value: var) !void {113pub fn writeILEB128(writer: var, int_value: var) !void {
124 const T = @TypeOf(int_value);114 const T = @TypeOf(int_value);
125 const S = if (T.bit_count < 8) i8 else T;115 const S = if (T.bit_count < 8) i8 else T;
...@@ -141,22 +131,19 @@ pub fn writeILEB128(writer: var, int_value: var) !void {...@@ -141,22 +131,19 @@ pub fn writeILEB128(writer: var, int_value: var) !void {
141 }131 }
142}132}
143133
144///Read a single singed LEB128 integer from the given memory as type T.134pub fn readILEB128Mem(comptime T: type, ptr: *[*]const u8) !T {
145/// The provided slice reference will be updated to point to the byte after the last byte read.135 const max_group = (T.bit_count + 6) / 7;
146pub fn readILEB128Mem(comptime T: type, ptr: *[]const u8) !T {136 var buf = std.io.fixedBufferStream(ptr.*[0 .. max_group + 1]);
147 var buf = std.io.fixedBufferStream(ptr.*);
148 const value = try readILEB128(T, buf.reader());137 const value = try readILEB128(T, buf.reader());
149 ptr.*.ptr += buf.pos;138 ptr.* += @intCast(usize, try buf.getPos());
150 return value;139 return value;
151}140}
152141
153///Write a single signed LEB128 integer to the given memory as unsigned LEB128,
154/// returning the number of bytes written.
155pub fn writeILEB128Mem(ptr: []u8, int_value: var) !usize {142pub fn writeILEB128Mem(ptr: []u8, int_value: var) !usize {
156 const T = @TypeOf(int_value);143 const T = @TypeOf(int_value);
157 var buf = std.io.fixedBufferStream(ptr);144 var buf = std.io.fixedBufferStream(ptr);
158 try writeILEB128(buf.writer(), int_value);145 try writeILEB128(buf.writer(), int_value);
159 return buf.pos;146 return try buf.getPos();
160}147}
161148
162//tests149//tests
...@@ -173,7 +160,7 @@ fn test_read_stream_uleb128(comptime T: type, encoded: []const u8) !T {...@@ -173,7 +160,7 @@ fn test_read_stream_uleb128(comptime T: type, encoded: []const u8) !T {
173fn test_read_ileb128(comptime T: type, encoded: []const u8) !T {160fn test_read_ileb128(comptime T: type, encoded: []const u8) !T {
174 var reader = std.io.fixedBufferStream(encoded);161 var reader = std.io.fixedBufferStream(encoded);
175 const v1 = try readILEB128(T, reader.reader());162 const v1 = try readILEB128(T, reader.reader());
176 var in_ptr = encoded;163 var in_ptr = encoded.ptr;
177 const v2 = try readILEB128Mem(T, &in_ptr);164 const v2 = try readILEB128Mem(T, &in_ptr);
178 testing.expectEqual(v1, v2);165 testing.expectEqual(v1, v2);
179 return v1;166 return v1;
...@@ -182,7 +169,7 @@ fn test_read_ileb128(comptime T: type, encoded: []const u8) !T {...@@ -182,7 +169,7 @@ fn test_read_ileb128(comptime T: type, encoded: []const u8) !T {
182fn test_read_uleb128(comptime T: type, encoded: []const u8) !T {169fn test_read_uleb128(comptime T: type, encoded: []const u8) !T {
183 var reader = std.io.fixedBufferStream(encoded);170 var reader = std.io.fixedBufferStream(encoded);
184 const v1 = try readULEB128(T, reader.reader());171 const v1 = try readULEB128(T, reader.reader());
185 var in_ptr = encoded;172 var in_ptr = encoded.ptr;
186 const v2 = try readULEB128Mem(T, &in_ptr);173 const v2 = try readULEB128Mem(T, &in_ptr);
187 testing.expectEqual(v1, v2);174 testing.expectEqual(v1, v2);
188 return v1;175 return v1;
...@@ -190,7 +177,7 @@ fn test_read_uleb128(comptime T: type, encoded: []const u8) !T {...@@ -190,7 +177,7 @@ fn test_read_uleb128(comptime T: type, encoded: []const u8) !T {
190177
191fn test_read_ileb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) !void {178fn test_read_ileb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) !void {
192 var reader = std.io.fixedBufferStream(encoded);179 var reader = std.io.fixedBufferStream(encoded);
193 var in_ptr = encoded;180 var in_ptr = encoded.ptr;
194 var i: usize = 0;181 var i: usize = 0;
195 while (i < N) : (i += 1) {182 while (i < N) : (i += 1) {
196 const v1 = try readILEB128(T, reader.reader());183 const v1 = try readILEB128(T, reader.reader());
...@@ -201,7 +188,7 @@ fn test_read_ileb128_seq(comptime T: type, comptime N: usize, encoded: []const u...@@ -201,7 +188,7 @@ fn test_read_ileb128_seq(comptime T: type, comptime N: usize, encoded: []const u
201188
202fn test_read_uleb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) !void {189fn test_read_uleb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) !void {
203 var reader = std.io.fixedBufferStream(encoded);190 var reader = std.io.fixedBufferStream(encoded);
204 var in_ptr = encoded;191 var in_ptr = encoded.ptr;
205 var i: usize = 0;192 var i: usize = 0;
206 while (i < N) : (i += 1) {193 while (i < N) : (i += 1) {
207 const v1 = try readULEB128(T, reader.reader());194 const v1 = try readULEB128(T, reader.reader());
...@@ -298,6 +285,8 @@ test "deserialize unsigned LEB128" {...@@ -298,6 +285,8 @@ test "deserialize unsigned LEB128" {
298fn test_write_leb128(value: var) !void {285fn test_write_leb128(value: var) !void {
299 const T = @TypeOf(value);286 const T = @TypeOf(value);
300287
288 if (T.bit_count == 0) std.debug.warn("{}\n", .{@typeName(T)});
289
301 const writeStream = if (T.is_signed) writeILEB128 else writeULEB128;290 const writeStream = if (T.is_signed) writeILEB128 else writeULEB128;
302 const writeMem = if (T.is_signed) writeILEB128Mem else writeULEB128Mem;291 const writeMem = if (T.is_signed) writeILEB128Mem else writeULEB128Mem;
303 const readStream = if (T.is_signed) readILEB128 else readULEB128;292 const readStream = if (T.is_signed) readILEB128 else readULEB128;
...@@ -335,13 +324,13 @@ fn test_write_leb128(value: var) !void {...@@ -335,13 +324,13 @@ fn test_write_leb128(value: var) !void {
335324
336 //mem read325 //mem read
337 var buf_ref: []u8 = buf[0..];326 var buf_ref: []u8 = buf[0..];
338 const mr = try readMem(T, &buf_ref);327 const mr = try readMem(T, &buf_ref.ptr);
339 testing.expect(@ptrToInt(buf_ref.ptr) - @ptrToInt(&buf) == w2_pos);328 testing.expect(@ptrToInt(buf_ref.ptr) - @ptrToInt(&buf) == w2_pos);
340 testing.expect(mr == value);329 testing.expect(mr == value);
341330
342 //bigger type mem read331 //bigger type mem read
343 buf_ref = buf[0..];332 buf_ref = buf[0..];
344 const bmr = try readMem(T, &buf_ref);333 const bmr = try readMem(T, &buf_ref.ptr);
345 testing.expect(@ptrToInt(buf_ref.ptr) - @ptrToInt(&buf) == w2_pos);334 testing.expect(@ptrToInt(buf_ref.ptr) - @ptrToInt(&buf) == w2_pos);
346 testing.expect(bmr == value);335 testing.expect(bmr == value);
347}336}