1//! Machine Intermediate Representation.
2//! This data is produced by x86_64 Codegen and consumed by x86_64 Isel.
3//! These instructions have a 1:1 correspondence with machine code instructions
4//! for the target. MIR can be lowered to source-annotated textual assembly code
5//! instructions, or it can be lowered to machine code.
6//! The main purpose of MIR is to postpone the assignment of offsets until Isel,
7//! so that, for example, the smaller encodings of jump instructions can be used.
8
9instructions: std.MultiArrayList(Inst).Slice,
10/// The meaning of this data is determined by `Inst.Tag` value.
11extra: []const u32,
12string_bytes: []const u8,
13locals: []const Local,
14table: []const Inst.Index,
15/// Optional data which, when present, can be used to accelerate encoding speed.
16memoized_encodings: []const u0 = &.{},
17frame_locs: std.MultiArrayList(FrameLoc).Slice,
18
19pub const Inst = struct {
20 tag: Tag,
21 ops: Ops,
22 data: Data,
23
24 pub const Index = u32;
25
26 pub const Fixes = enum(u8) {
27 /// ___
28 @"_",
29
30 /// ___ 0
31 _0,
32 /// ___ 1
33 _1,
34 /// ___ 2
35 _2,
36 /// ___ 3
37 _3,
38 /// ___ 4
39 _4,
40
41 /// ___ Demote
42 _demote,
43 /// ___ Flush
44 _flush,
45 /// ___ Flush Optimized
46 _flushopt,
47 /// ___ Instructions With T0 Hint
48 _it0,
49 /// ___ Instructions With T0 Hint
50 _it1,
51 /// ___ With NTA Hint
52 _nta,
53 /// System Call ___
54 sys_,
55 /// ___ With T0 Hint
56 _t0,
57 /// ___ With T1 Hint
58 _t1,
59 /// ___ With T2 Hint
60 _t2,
61 /// ___ Write Back
62 _wb,
63 /// ___ With Intent to Write and T1 Hint
64 _wt1,
65
66 /// ___ crement Shadow Stack Pointer Doubleword
67 _csspd,
68 /// ___ crement Shadow Stack Pointer Quadword
69 _csspq,
70 /// ___ FS Segment Base
71 _fsbase,
72 /// ___ GS
73 _gs,
74 /// ___ GS Segment Base
75 _gsbase,
76 /// ___ Model Specific Register
77 _msr,
78 /// ___ MXCSR
79 _mxcsr,
80 /// ___ Processor ID
81 _pid,
82 /// ___ Protection Key Rights For User Pages
83 _pkru,
84 /// ___ Performance-Monitoring Counters
85 _pmc,
86 /// ___ Random Number
87 _rand,
88 /// ___ r Busy Flag in a Supervisor Shadow Stack token
89 _rssbsy,
90 /// ___ Random Seed
91 _seed,
92 /// ___ Shadow Stack Doubleword
93 _ssd,
94 /// ___ Shadow Stack Quadword
95 _ssq,
96 /// ___ Shadow Stack Pointer Doubleword
97 _sspd,
98 /// ___ Shadow Stack Pointer Quadword
99 _sspq,
100 /// ___ Time-Stamp Counter
101 _tsc,
102 /// ___ Time-Stamp Counter And Processor ID
103 _tscp,
104 /// ___ User Shadow Stack Doubleword
105 _ussd,
106 /// ___ User Shadow Stack Quadword
107 _ussq,
108 /// VEX-Encoded ___ MXCSR
109 v_mxcsr,
110
111 /// Byte ___
112 b_,
113 /// Interrupt ___
114 /// Integer ___
115 i_,
116 /// Interrupt ___ Word
117 i_w,
118 /// Interrupt ___ Doubleword
119 i_d,
120 /// Interrupt ___ Quadword
121 i_q,
122 /// User-Interrupt ___
123 ui_,
124
125 /// ___ mp
126 _mp,
127 /// ___ if CX register is 0
128 _cxz,
129 /// ___ if ECX register is 0
130 _ecxz,
131 /// ___ if RCX register is 0
132 _rcxz,
133
134 /// ___ Addition
135 _a,
136 /// ___ Subtraction
137 _s,
138 /// ___ Multiply
139 _m,
140 /// ___ Division
141 _d,
142
143 /// ___ Without Affecting Flags
144 _x,
145 /// ___ Left
146 _l,
147 /// ___ Left Double
148 _ld,
149 /// ___ Left Without Affecting Flags
150 _lx,
151 /// ___ Mask
152 _msk,
153 /// ___ Right
154 /// ___ For Reading
155 /// ___ Register
156 _r,
157 /// ___ Right Double
158 _rd,
159 /// ___ Right Without Affecting Flags
160 _rx,
161
162 /// ___ Forward
163 _f,
164 /// ___ Reverse
165 //_r,
166
167 /// ___ Above
168 //_a,
169 /// ___ Above Or Equal
170 _ae,
171 /// ___ Below
172 _b,
173 /// ___ Below Or Equal
174 /// ___ Big Endian
175 _be,
176 /// ___ Carry
177 /// ___ Carry Flag
178 _c,
179 /// ___ Equal
180 _e,
181 /// ___ Greater
182 _g,
183 /// ___ Greater Or Equal
184 _ge,
185 /// ___ Less
186 //_l,
187 /// ___ Less Or Equal
188 _le,
189 /// ___ Not Above
190 _na,
191 /// ___ Not Above Or Equal
192 _nae,
193 /// ___ Not Below
194 _nb,
195 /// ___ Not Below Or Equal
196 _nbe,
197 /// ___ Not Carry
198 _nc,
199 /// ___ Not Equal
200 _ne,
201 /// ___ Not Greater
202 _ng,
203 /// ___ Not Greater Or Equal
204 _nge,
205 /// ___ Not Less
206 _nl,
207 /// ___ Not Less Or Equal
208 _nle,
209 /// ___ Not Overflow
210 _no,
211 /// ___ Not Parity
212 _np,
213 /// ___ Not Sign
214 _ns,
215 /// ___ Not Zero
216 _nz,
217 /// ___ Overflow
218 _o,
219 /// ___ Parity
220 _p,
221 /// ___ Parity Even
222 _pe,
223 /// ___ Parity Odd
224 _po,
225 /// ___ Sign
226 //_s,
227 /// ___ Zero
228 _z,
229 /// ___ Alignment Check Flag
230 _ac,
231 /// ___ Direction Flag
232 //_d,
233 /// ___ Interrupt Flag
234 _i,
235 /// ___ Task-Switched Flag In CR0
236 _ts,
237 /// ___ User Interrupt Flag
238 _ui,
239
240 /// ___ Byte
241 //_b,
242 /// ___ Word
243 /// ___ For Writing
244 /// ___ With Intent to Write
245 _w,
246 /// ___ Doubleword
247 //_d,
248 /// ___ Double Quadword to Quadword
249 _dq2q,
250 /// ___ QuadWord
251 _q,
252 /// ___ Quadword to Double Quadword
253 _q2dq,
254
255 /// ___ String
256 //_s,
257 /// ___ String Byte
258 _sb,
259 /// ___ String Word
260 _sw,
261 /// ___ String Doubleword
262 _sd,
263 /// ___ String Quadword
264 _sq,
265
266 /// Repeat ___ String
267 @"rep _s",
268 /// Repeat ___ String Byte
269 @"rep _sb",
270 /// Repeat ___ String Word
271 @"rep _sw",
272 /// Repeat ___ String Doubleword
273 @"rep _sd",
274 /// Repeat ___ String Quadword
275 @"rep _sq",
276
277 /// Repeat Equal ___ String
278 @"repe _s",
279 /// Repeat Equal ___ String Byte
280 @"repe _sb",
281 /// Repeat Equal ___ String Word
282 @"repe _sw",
283 /// Repeat Equal ___ String Doubleword
284 @"repe _sd",
285 /// Repeat Equal ___ String Quadword
286 @"repe _sq",
287
288 /// Repeat Not Equal ___ String
289 @"repne _s",
290 /// Repeat Not Equal ___ String Byte
291 @"repne _sb",
292 /// Repeat Not Equal ___ String Word
293 @"repne _sw",
294 /// Repeat Not Equal ___ String Doubleword
295 @"repne _sd",
296 /// Repeat Not Equal ___ String Quadword
297 @"repne _sq",
298
299 /// Repeat Not Zero ___ String
300 @"repnz _s",
301 /// Repeat Not Zero ___ String Byte
302 @"repnz _sb",
303 /// Repeat Not Zero ___ String Word
304 @"repnz _sw",
305 /// Repeat Not Zero ___ String Doubleword
306 @"repnz _sd",
307 /// Repeat Not Zero ___ String Quadword
308 @"repnz _sq",
309
310 /// Repeat Zero ___ String
311 @"repz _s",
312 /// Repeat Zero ___ String Byte
313 @"repz _sb",
314 /// Repeat Zero ___ String Word
315 @"repz _sw",
316 /// Repeat Zero ___ String Doubleword
317 @"repz _sd",
318 /// Repeat Zero ___ String Quadword
319 @"repz _sq",
320
321 /// Locked ___
322 @"lock _",
323 /// ___ And Complement
324 //_c,
325 /// Locked ___ And Complement
326 @"lock _c",
327 /// ___ And Reset
328 //_r,
329 /// Locked ___ And Reset
330 @"lock _r",
331 /// ___ And Set
332 //_s,
333 /// Locked ___ And Set
334 @"lock _s",
335 /// ___ 8 Bytes
336 _8b,
337 /// Locked ___ 8 Bytes
338 @"lock _8b",
339 /// ___ 16 Bytes
340 _16b,
341 /// Locked ___ 16 Bytes
342 @"lock _16b",
343
344 /// Float ___
345 f_,
346 /// Float ___ +1.0
347 /// Float ___ 1
348 f_1,
349 /// Float ___ Below
350 f_b,
351 /// Float ___ Below Or Equal
352 f_be,
353 /// Float ___ Control Word
354 f_cw,
355 /// Float ___ Equal
356 f_e,
357 /// Float ___ Environment
358 f_env,
359 /// Float ___ log_2(e)
360 f_l2e,
361 /// Float ___ log_2(10)
362 f_l2t,
363 /// Float ___ log_10(2)
364 f_lg2,
365 /// Float ___ log_e(2)
366 f_ln2,
367 /// Float ___ Not Below
368 f_nb,
369 /// Float ___ Not Below Or Equal
370 f_nbe,
371 /// Float ___ Not Equal
372 f_ne,
373 /// Float ___ Not Unordered
374 f_nu,
375 /// Float ___ Pop
376 f_p,
377 /// Float ___ +1
378 f_p1,
379 /// Float ___ π
380 f_pi,
381 /// Float ___ Pop Pop
382 f_pp,
383 /// Float ___ crement Stack-Top Pointer
384 f_cstp,
385 /// Float ___ Status Word
386 f_sw,
387 /// Float ___ Unordered
388 f_u,
389 /// Float ___ +0.0
390 f_z,
391 /// Float BCD ___
392 fb_,
393 /// Float BCD ___ Pop
394 fb_p,
395 /// Float And Integer ___
396 fi_,
397 /// Float And Integer ___ Pop
398 fi_p,
399 /// Float No Wait ___
400 fn_,
401 /// Float No Wait ___ Control Word
402 fn_cw,
403 /// Float No Wait ___ Environment
404 fn_env,
405 /// Float No Wait ___ status word
406 fn_sw,
407 /// Float Extended ___
408 fx_,
409 /// Float Extended ___ 64
410 fx_64,
411
412 /// ___ in 32-bit and Compatibility Mode
413 _32,
414 /// ___ in 64-bit Mode
415 _64,
416
417 /// Packed ___
418 p_,
419 /// Packed ___ Byte
420 p_b,
421 /// Packed ___ Word
422 p_w,
423 /// Packed ___ Doubleword
424 p_d,
425 /// Packed ___ Quadword
426 p_q,
427 /// Packed ___ Double Quadword
428 /// Packed ___ Doubleword to Quadword
429 p_dq,
430 /// Packed ___ Unsigned Doubleword to Quadword
431 p_udq,
432 /// Packed Carry-Less ___ Quadword to Double Quadword
433 pcl_qdq,
434 /// Packed Half ___ Doubleword
435 ph_d,
436 /// Packed Half ___ Saturate Word
437 ph_sw,
438 /// Packed Half ___ Word
439 ph_w,
440 /// ___ Aligned Packed Integer Values
441 _dqa,
442 /// ___ Unaligned Packed Integer Values
443 _dqu,
444
445 /// ___ Scalar Single-Precision Values
446 _ss,
447 /// ___ Packed Single-Precision Values
448 _ps,
449 /// ___ Scalar Double-Precision Values
450 //_sd,
451 /// ___ Packed Double-Precision Values
452 _pd,
453 /// Half ___ Packed Single-Precision Values
454 h_ps,
455 /// Half ___ Packed Double-Precision Values
456 h_pd,
457
458 /// ___ Internal Caches
459 //_d,
460 /// ___ TLB Entries
461 _lpg,
462 /// ___ Process-Context Identifier
463 _pcid,
464
465 /// Load ___
466 l_,
467 /// Memory ___
468 m_,
469 /// Store ___
470 s_,
471 /// Timed ___
472 t_,
473 /// User Level Monitor ___
474 um_,
475
476 /// VEX-Encoded ___
477 v_,
478 /// VEX-Encoded ___ Byte
479 v_b,
480 /// VEX-Encoded ___ Word
481 v_w,
482 /// VEX-Encoded ___ Doubleword
483 v_d,
484 /// VEX-Encoded ___ Quadword
485 v_q,
486 /// VEX-Encoded ___ Aligned Packed Integer Values
487 v_dqa,
488 /// VEX-Encoded ___ Unaligned Packed Integer Values
489 v_dqu,
490 /// VEX-Encoded ___ Integer Data
491 v_i128,
492 /// VEX-Encoded Packed ___
493 vp_,
494 /// VEX-Encoded Packed ___ Byte
495 vp_b,
496 /// VEX-Encoded Packed ___ Word
497 vp_w,
498 /// VEX-Encoded Packed ___ Doubleword
499 vp_d,
500 /// VEX-Encoded Packed ___ Quadword
501 vp_q,
502 /// VEX-Encoded Packed ___ Double Quadword
503 /// VEX-Encoded Packed ___ Doubleword to Quadword
504 vp_dq,
505 /// VEX-Encoded Packed ___ Unsigned Doubleword to Quadword
506 vp_udq,
507 /// VEx-Encoded Packed Carry-Less ___ Quadword to Double Quadword
508 vpcl_qdq,
509 /// VEX-Encoded Packed Half ___ Doubleword
510 vph_d,
511 /// VEX-Encoded Packed Half ___ Saturate Word
512 vph_sw,
513 /// VEX-Encoded Packed Half ___ Word
514 vph_w,
515 /// VEX-Encoded ___ Scalar Single-Precision Values
516 v_ss,
517 /// VEX-Encoded ___ Packed Single-Precision Values
518 v_ps,
519 /// VEX-Encoded ___ Scalar Double-Precision Values
520 v_sd,
521 /// VEX-Encoded ___ Packed Double-Precision Values
522 v_pd,
523 /// VEX-Encoded ___ 128-Bits Of Floating-Point Data
524 v_f128,
525 /// VEX-Encoded Half ___ Packed Single-Precision Values
526 vh_ps,
527 /// VEX-Encoded Half ___ Packed Double-Precision Values
528 vh_pd,
529
530 /// ___ 128-bit key with key locker
531 _128,
532 /// ___ 256-bit key with key locker
533 _256,
534 /// ___ with key locker using 128-bit key
535 _128kl,
536 /// ___ with key locker using 256-bit key
537 _256kl,
538 /// ___ with key locker on 8 blocks using 128-bit key
539 _wide128kl,
540 /// ___ with key locker on 8 blocks using 256-bit key
541 _wide256kl,
542
543 /// Mask ___ Byte
544 k_b,
545 /// Mask ___ Word
546 k_w,
547 /// Mask ___ Doubleword
548 k_d,
549 /// Mask ___ Quadword
550 k_q,
551
552 pub fn fromCond(cc: bits.Condition) Fixes {
553 return switch (cc) {
554 inline else => |cc_tag| @field(Fixes, "_" ++ @tagName(cc_tag)),
555 .z_and_np, .nz_or_p => unreachable,
556 };
557 }
558 };
559
560 pub const Tag = enum(u8) {
561 // General-purpose
562 /// ASCII adjust al after addition
563 /// ASCII adjust ax before division
564 /// ASCII adjust ax after multiply
565 /// ASCII adjust al after subtraction
566 aa,
567 /// Add with carry
568 /// Unsigned integer addition of two operands with carry flag
569 adc,
570 /// Add
571 /// Add packed integers
572 /// Add packed single-precision floating-point values
573 /// Add scalar single-precision floating-point values
574 /// Add packed double-precision floating-point values
575 /// Add scalar double-precision floating-point values
576 /// Packed single-precision floating-point horizontal add
577 /// Packed double-precision floating-point horizontal add
578 /// Packed horizontal add
579 /// Packed horizontal add and saturate
580 add,
581 /// Logical and
582 /// Bitwise logical and of packed single-precision floating-point values
583 /// Bitwise logical and of packed double-precision floating-point values
584 @"and",
585 /// Adjust RPL field of segment selector
586 arpl,
587 /// Bit scan forward
588 /// Bit scan reverse
589 bs,
590 /// Byte swap
591 /// Swap GS base register
592 swap,
593 /// Bit test
594 /// Bit test and complement
595 /// Bit test and reset
596 /// Bit test and set
597 bt,
598 /// Check array index against bounds
599 bound,
600 /// Call
601 /// Fast system call
602 call,
603 /// Convert byte to word
604 cbw,
605 /// Convert doubleword to quadword
606 cdq,
607 /// Convert doubleword to quadword
608 cdqe,
609 /// Clear AC flag in EFLAGS register
610 /// Clear carry flag
611 /// Clear direction flag
612 /// Clear interrupt flag
613 /// Clear task-switched flag in CR0
614 /// Clear user interrupt flag
615 /// Cache line demote
616 /// Flush cache line
617 /// Flush cache line optimized
618 /// Clear busy flag in a supervisor shadow stack token
619 /// Cache line write back
620 cl,
621 /// Complement carry flag
622 cmc,
623 /// Conditional move
624 cmov,
625 /// Logical compare
626 /// Compare string
627 /// Compare scalar single-precision floating-point values
628 /// Compare scalar double-precision floating-point values
629 cmp,
630 /// Compare and exchange
631 /// Compare and exchange bytes
632 cmpxchg,
633 /// CPU identification
634 cpuid,
635 /// Convert doubleword to quadword
636 cqo,
637 /// Convert word to doubleword
638 cwd,
639 /// Convert word to doubleword
640 cwde,
641 /// Decimal adjust AL after addition
642 /// Decimal adjust AL after subtraction
643 da,
644 /// Decrement by 1
645 /// Decrement stack-top pointer
646 /// Decrement shadow stack pointer
647 de,
648 /// Unsigned division
649 /// Signed division
650 /// Divide
651 /// Divide packed single-precision floating-point values
652 /// Divide scalar single-precision floating-point values
653 /// Divide packed double-precision floating-point values
654 /// Divide scalar double-precision floating-point values
655 div,
656 /// Terminate and indirect branch in 32-bit and compatibility mode
657 /// Terminate and indirect branch in 64-bit mode
658 endbr,
659 /// Enqueue command
660 /// Enqueue command supervisor
661 enqcmd,
662 /// Make stack frame for procedure parameters
663 /// Fast system call
664 enter,
665 /// Fast return from fast system call
666 exit,
667 /// Load fence
668 /// Memory fence
669 /// Store fence
670 fence,
671 /// Halt
672 hlt,
673 /// History reset
674 hreset,
675 /// Input from port
676 /// Input from port to string
677 /// Increment by 1
678 /// Increment stack-top pointer
679 /// Increment shadow stack pointer
680 in,
681 /// Call to interrupt procedure
682 int,
683 /// Invalidate internal caches
684 /// Invalidate TLB entries
685 /// Invalidate process-context identifier
686 inv,
687 /// Conditional jump
688 /// Jump
689 j,
690 /// Load status flags into AH register
691 lahf,
692 /// Load access right byte
693 lar,
694 /// Load effective address
695 lea,
696 /// High level procedure exit
697 leave,
698 /// Load global descriptor table register
699 lgdt,
700 /// Load interrupt descriptor table register
701 lidt,
702 /// Load local descriptor table register
703 lldt,
704 /// Load machine status word
705 lmsw,
706 /// Load string
707 lod,
708 /// Loop according to ECX counter
709 loop,
710 /// Load segment limit
711 lsl,
712 /// Load task register
713 ltr,
714 /// Count the number of leading zero bits
715 lzcnt,
716 /// Move
717 /// Move data from string to string
718 /// Move data after swapping bytes
719 /// Move scalar single-precision floating-point value
720 /// Move scalar double-precision floating-point value
721 /// Move doubleword
722 /// Move quadword
723 /// Move aligned packed integer values
724 /// Move unaligned packed integer values
725 /// Move quadword from XMM to MMX technology register
726 /// Move quadword from MMX technology to XMM register
727 mov,
728 /// Move with sign extension
729 movsx,
730 /// Move with zero extension
731 movzx,
732 /// Multiply
733 /// Signed multiplication
734 /// Multiply packed single-precision floating-point values
735 /// Multiply scalar single-precision floating-point values
736 /// Multiply packed double-precision floating-point values
737 /// Multiply scalar double-precision floating-point values
738 /// Multiply packed unsigned doubleword integers
739 /// Multiply packed doubleword integers
740 /// Carry-less multiplication quadword
741 mul,
742 /// Two's complement negation
743 neg,
744 /// No-op
745 /// No operation
746 nop,
747 /// One's complement negation
748 not,
749 /// Logical or
750 /// Bitwise logical or of packed single-precision floating-point values
751 /// Bitwise logical or of packed double-precision floating-point values
752 @"or",
753 /// Output to port
754 /// Output string to port
755 out,
756 /// Spin loop hint
757 /// Timed pause
758 pause,
759 /// Pop
760 pop,
761 /// Return the count of number of bits set to 1
762 popcnt,
763 /// Pop stack into EFLAGS register
764 popf,
765 /// Push
766 push,
767 /// Push EFLAGS register onto the stack
768 pushf,
769 /// Rotate left through carry
770 /// Rotate right through carry
771 rc,
772 /// Read FS segment base
773 /// Read GS segment base
774 /// Read from model specific register
775 /// Read processor ID
776 /// Read protection key rights for user pages
777 /// Read performance-monitoring counters
778 /// Read random number
779 /// Read random seed
780 /// Read shadow stack pointer
781 /// Read time-stamp counter
782 /// Read time-stamp counter and processor ID
783 rd,
784 /// Return
785 /// Return from fast system call
786 /// Interrupt return
787 /// User-interrupt return
788 ret,
789 /// Rotate left
790 /// Rotate right
791 /// Rotate right logical without affecting flags
792 ro,
793 /// Resume from system management mode
794 rsm,
795 /// Arithmetic shift left
796 /// Arithmetic shift right
797 /// Shift left arithmetic without affecting flags
798 sa,
799 /// Store AH into flags
800 sahf,
801 /// Integer subtraction with borrow
802 sbb,
803 /// Scan string
804 sca,
805 /// Send user interprocessor interrupt
806 senduipi,
807 /// Serialize instruction execution
808 serialize,
809 /// Set byte on condition
810 set,
811 /// Logical shift left
812 /// Double precision shift left
813 /// Logical shift right
814 /// Double precision shift right
815 /// Shift left logical without affecting flags
816 /// Shift right logical without affecting flags
817 sh,
818 /// Store interrupt descriptor table register
819 sidt,
820 /// Store local descriptor table register
821 sldt,
822 /// Store machine status word
823 smsw,
824 /// Subtract
825 /// Subtract packed integers
826 /// Subtract packed single-precision floating-point values
827 /// Subtract scalar single-precision floating-point values
828 /// Subtract packed double-precision floating-point values
829 /// Subtract scalar double-precision floating-point values
830 /// Packed single-precision floating-point horizontal subtract
831 /// Packed double-precision floating-point horizontal subtract
832 /// Packed horizontal subtract
833 /// Packed horizontal subtract and saturate
834 sub,
835 /// Set carry flag
836 /// Set direction flag
837 /// Set interrupt flag
838 /// Store binary coded decimal integer and pop
839 /// Store floating-point value
840 /// Store integer
841 /// Store x87 FPU control word
842 /// Store x87 FPU environment
843 /// Store x87 FPU status word
844 /// Store MXCSR register state
845 st,
846 /// Store string
847 sto,
848 /// Test condition
849 /// Logical compare
850 /// Packed bit test
851 @"test",
852 /// Undefined instruction
853 ud,
854 /// User level set up monitor address
855 umonitor,
856 /// Verify a segment for reading
857 /// Verify a segment for writing
858 ver,
859 /// Write to model specific register
860 /// Write to model specific register
861 /// Write to model specific register
862 /// Write to shadow stack
863 /// Write to user shadow stack
864 wr,
865 /// Exchange and add
866 xadd,
867 /// Exchange register/memory with register
868 /// Exchange register contents
869 xch,
870 /// Get value of extended control register
871 xgetbv,
872 /// Table look-up translation
873 xlat,
874 /// Logical exclusive-or
875 /// Bitwise logical xor of packed single-precision floating-point values
876 /// Bitwise logical xor of packed double-precision floating-point values
877 xor,
878
879 // X87
880 /// Compute 2^x-1
881 @"2xm1",
882 /// Absolute value
883 abs,
884 /// Change sign
885 chs,
886 /// Clear exceptions
887 clex,
888 /// Compare floating-point values
889 com,
890 /// Compare floating-point values and set EFLAGS
891 /// Compare scalar ordered single-precision floating-point values
892 /// Compare scalar ordered double-precision floating-point values
893 comi,
894 /// Cosine
895 cos,
896 /// Reverse divide
897 divr,
898 /// Free floating-point register
899 free,
900 /// Initialize floating-point unit
901 init,
902 /// Load binary coded decimal integer
903 /// Load floating-point value
904 /// Load integer
905 /// Load constant
906 /// Load x87 FPU control word
907 /// Load x87 FPU environment
908 /// Load MXCSR register state
909 ld,
910 /// Partial arctangent
911 patan,
912 /// Partial remainder
913 prem,
914 /// Partial tangent
915 ptan,
916 /// Round to integer
917 rndint,
918 /// Restore x87 FPU state
919 /// Restore x87 FPU, MMX, XMM, and MXCSR state
920 rstor,
921 /// Store x87 FPU state
922 /// Save x87 FPU, MMX technology, and MXCSR state
923 save,
924 /// Scale
925 scale,
926 /// Sine
927 sin,
928 /// Sine and cosine
929 sincos,
930 /// Square root
931 /// Square root of packed single-precision floating-point values
932 /// Square root of scalar single-precision floating-point value
933 /// Square root of packed double-precision floating-point values
934 /// Square root of scalar double-precision floating-point value
935 sqrt,
936 /// Store integer with truncation
937 stt,
938 /// Reverse subtract
939 subr,
940 /// Test
941 tst,
942 /// Unordered compare floating-point values
943 ucom,
944 /// Unordered compare floating-point values and set EFLAGS
945 /// Unordered compare scalar single-precision floating-point values
946 /// Unordered compare scalar double-precision floating-point values
947 ucomi,
948 /// Wait
949 /// User level monitor wait
950 wait,
951 /// Examine floating-point
952 xam,
953 /// Extract exponent and significand
954 xtract,
955 /// Compute y * log2x
956 /// Compute y * log2(x + 1)
957 yl2x,
958
959 // MMX
960 /// Pack with signed saturation
961 ackssw,
962 /// Pack with signed saturation
963 ackssd,
964 /// Pack with unsigned saturation
965 ackusw,
966 /// Add packed signed integers with signed saturation
967 adds,
968 /// Add packed unsigned integers with unsigned saturation
969 addus,
970 /// Logical and not
971 /// Bitwise logical and not of packed single-precision floating-point values
972 /// Bitwise logical and not of packed double-precision floating-point values
973 andn,
974 /// Compare packed data for equal
975 cmpeq,
976 /// Compare packed data for greater than
977 cmpgt,
978 /// Empty MMX technology state
979 emms,
980 /// Multiply and add packed signed and unsigned bytes
981 maddubs,
982 /// Multiply and add packed integers
983 maddw,
984 /// Multiply packed signed integers and store low result
985 mull,
986 /// Multiply packed signed integers and store high result
987 mulh,
988 /// Shift packed data left logical
989 sll,
990 /// Shift packed data right arithmetic
991 sra,
992 /// Shift packed data right logical
993 srl,
994 /// Subtract packed signed integers with signed saturation
995 subs,
996 /// Subtract packed unsigned integers with unsigned saturation
997 subus,
998 /// Unpack high data
999 unpckhbw,
1000 /// Unpack high data
1001 unpckhdq,
1002 /// Unpack high data
1003 unpckhwd,
1004 /// Unpack low data
1005 unpcklbw,
1006 /// Unpack low data
1007 unpckldq,
1008 /// Unpack low data
1009 unpcklwd,
1010
1011 // SSE
1012 /// Average packed integers
1013 avg,
1014 /// Convert packed doubleword integers to packed single-precision floating-point values
1015 /// Convert packed doubleword integers to packed double-precision floating-point values
1016 cvtpi2,
1017 /// Convert packed single-precision floating-point values to packed doubleword integers
1018 cvtps2pi,
1019 /// Convert doubleword integer to scalar single-precision floating-point value
1020 /// Convert doubleword integer to scalar double-precision floating-point value
1021 cvtsi2,
1022 /// Convert scalar single-precision floating-point value to doubleword integer
1023 cvtss2si,
1024 /// Convert with truncation packed single-precision floating-point values to packed doubleword integers
1025 cvttps2pi,
1026 /// Convert with truncation scalar single-precision floating-point value to doubleword integer
1027 cvttss2si,
1028 /// Extract byte
1029 /// Extract word
1030 /// Extract doubleword
1031 /// Extract quadword
1032 extr,
1033 /// Insert byte
1034 /// Insert word
1035 /// Insert doubleword
1036 /// Insert quadword
1037 insr,
1038 /// Maximum of packed single-precision floating-point values
1039 /// Maximum of scalar single-precision floating-point values
1040 /// Maximum of packed double-precision floating-point values
1041 /// Maximum of scalar double-precision floating-point values
1042 max,
1043 /// Maximum of packed signed integers
1044 maxs,
1045 /// Maximum of packed unsigned integers
1046 maxu,
1047 /// Minimum of packed single-precision floating-point values
1048 /// Minimum of scalar single-precision floating-point values
1049 /// Minimum of packed double-precision floating-point values
1050 /// Minimum of scalar double-precision floating-point values
1051 min,
1052 /// Minimum of packed signed integers
1053 mins,
1054 /// Minimum of packed unsigned integers
1055 minu,
1056 /// Move aligned packed single-precision floating-point values
1057 /// Move aligned packed double-precision floating-point values
1058 mova,
1059 /// Move high packed single-precision floating-point values
1060 /// Move high packed double-precision floating-point values
1061 movh,
1062 /// Move packed single-precision floating-point values high to low
1063 movhl,
1064 /// Move low packed single-precision floating-point values
1065 /// Move low packed double-precision floating-point values
1066 movl,
1067 /// Move packed single-precision floating-point values low to high
1068 movlh,
1069 /// Move byte mask
1070 /// Extract packed single precision floating-point sign mask
1071 /// Extract packed double precision floating-point sign mask
1072 movmsk,
1073 /// Move unaligned packed single-precision floating-point values
1074 /// Move unaligned packed double-precision floating-point values
1075 movu,
1076 /// Multiply packed unsigned integers and store high result
1077 mulhu,
1078 /// Prefetch data into caches
1079 /// Prefetch data into caches with intent to write
1080 prefetch,
1081 /// Compute sum of absolute differences
1082 sadb,
1083 /// Packed interleave shuffle of quadruplets of single-precision floating-point values
1084 /// Packed interleave shuffle of pairs of double-precision floating-point values
1085 /// Shuffle packed doublewords
1086 /// Shuffle packed words
1087 shuf,
1088 /// Unpack and interleave high packed single-precision floating-point values
1089 /// Unpack and interleave high packed double-precision floating-point values
1090 unpckh,
1091 /// Unpack and interleave low packed single-precision floating-point values
1092 /// Unpack and interleave low packed double-precision floating-point values
1093 unpckl,
1094
1095 // SSE2
1096 /// Convert packed doubleword integers to packed single-precision floating-point values
1097 /// Convert packed doubleword integers to packed double-precision floating-point values
1098 cvtdq2,
1099 /// Convert packed double-precision floating-point values to packed doubleword integers
1100 cvtpd2dq,
1101 /// Convert packed double-precision floating-point values to packed doubleword integers
1102 cvtpd2pi,
1103 /// Convert packed double-precision floating-point values to packed single-precision floating-point values
1104 cvtpd2,
1105 /// Convert packed single-precision floating-point values to packed doubleword integers
1106 cvtps2dq,
1107 /// Convert packed single-precision floating-point values to packed double-precision floating-point values
1108 cvtps2,
1109 /// Convert scalar double-precision floating-point value to doubleword integer
1110 cvtsd2si,
1111 /// Convert scalar double-precision floating-point value to scalar single-precision floating-point value
1112 cvtsd2,
1113 /// Convert scalar single-precision floating-point value to scalar double-precision floating-point value
1114 cvtss2,
1115 /// Convert with truncation packed double-precision floating-point values to packed doubleword integers
1116 cvttpd2dq,
1117 /// Convert with truncation packed double-precision floating-point values to packed doubleword integers
1118 cvttpd2pi,
1119 /// Convert with truncation packed single-precision floating-point values to packed doubleword integers
1120 cvttps2dq,
1121 /// Convert with truncation scalar double-precision floating-point value to doubleword integer
1122 cvttsd2si,
1123 /// Galois field affine transformation inverse
1124 gf2p8affineinvq,
1125 /// Galois field affine transformation
1126 gf2p8affineq,
1127 /// Galois field multiply bytes
1128 gf2p8mul,
1129 /// Shuffle packed high words
1130 shufh,
1131 /// Shuffle packed low words
1132 shufl,
1133 /// Unpack high data
1134 unpckhqdq,
1135 /// Unpack low data
1136 unpcklqdq,
1137
1138 // SSE3
1139 /// Packed single-precision floating-point add/subtract
1140 /// Packed double-precision floating-point add/subtract
1141 addsub,
1142 /// Replicate double floating-point values
1143 movddup,
1144 /// Replicate single floating-point values
1145 movshdup,
1146 /// Replicate single floating-point values
1147 movsldup,
1148
1149 // SSSE3
1150 /// Packed align right
1151 alignr,
1152 /// Packed multiply high with round and scale
1153 mulhrs,
1154 /// Packed sign
1155 sign,
1156
1157 // SSE4.1
1158 /// Pack with unsigned saturation
1159 ackusd,
1160 /// Blend packed single-precision floating-point values
1161 /// Blend scalar single-precision floating-point values
1162 /// Blend packed double-precision floating-point values
1163 /// Blend scalar double-precision floating-point values
1164 /// Blend packed dwords
1165 blend,
1166 /// Variable blend packed single-precision floating-point values
1167 /// Variable blend scalar single-precision floating-point values
1168 /// Variable blend packed double-precision floating-point values
1169 /// Variable blend scalar double-precision floating-point values
1170 blendv,
1171 /// Dot product of packed single-precision floating-point values
1172 /// Dot product of packed double-precision floating-point values
1173 dp,
1174 /// Extract packed floating-point values
1175 /// Extract packed integer values
1176 extract,
1177 /// Insert scalar single-precision floating-point value
1178 /// Insert packed floating-point values
1179 insert,
1180 /// Packed horizontal word minimum
1181 minposu,
1182 /// Packed move with sign extend
1183 movsxb,
1184 movsxd,
1185 movsxw,
1186 /// Packed move with zero extend
1187 movzxb,
1188 movzxd,
1189 movzxw,
1190 /// Round packed single-precision floating-point values
1191 /// Round scalar single-precision floating-point value
1192 /// Round packed double-precision floating-point values
1193 /// Round scalar double-precision floating-point value
1194 round,
1195
1196 // SSE4.2
1197 /// Accumulate CRC32 value
1198 crc32,
1199
1200 // AES
1201 /// Perform one round of an AES decryption flow
1202 /// Perform ten rounds of AES decryption flow with key locker using 128-bit key
1203 /// Perform ten rounds of AES decryption flow with key locker using 256-bit key
1204 /// Perform ten rounds of AES decryption flow with key locker on 8 blocks using 128-bit key
1205 /// Perform ten rounds of AES decryption flow with key locker on 8 blocks using 256-bit key
1206 aesdec,
1207 /// Perform last round of an AES decryption flow
1208 aesdeclast,
1209 /// Perform one round of an AES encryption flow
1210 /// Perform ten rounds of AES encryption flow with key locker using 128-bit key
1211 /// Perform ten rounds of AES encryption flow with key locker using 256-bit key
1212 /// Perform ten rounds of AES encryption flow with key locker on 8 blocks using 128-bit key
1213 /// Perform ten rounds of AES encryption flow with key locker on 8 blocks using 256-bit key
1214 aesenc,
1215 /// Perform last round of an AES encryption flow
1216 aesenclast,
1217 /// Perform the AES InvMixColumn transformation
1218 aesimc,
1219 /// AES round key generation assist
1220 aeskeygenassist,
1221
1222 // SHA
1223 /// Perform four rounds of SHA1 operation
1224 sha1rnds,
1225 /// Calculate SHA1 state variable E after four rounds
1226 sha1nexte,
1227 /// Perform an intermediate calculation for the next four SHA1 message dwords
1228 /// Perform a final calculation for the next four SHA1 message dwords
1229 sha1msg,
1230 /// Perform an intermediate calculation for the next four SHA256 message dwords
1231 /// Perform a final calculation for the next four SHA256 message dwords
1232 sha256msg,
1233 /// Perform two rounds of SHA256 operation
1234 sha256rnds,
1235
1236 // AVX
1237 /// Load with broadcast floating-point data
1238 /// Load integer and broadcast
1239 broadcast,
1240 /// Conditional SIMD packed loads and stores
1241 /// Condition SIMD integer packed loads and stores
1242 maskmov,
1243 /// Permute floating-point values
1244 /// Permute integer values
1245 perm2,
1246 /// Permute in-lane pairs of double-precision floating-point values
1247 /// Permute in-lane quadruples of single-precision floating-point values
1248 permil,
1249
1250 // BMI
1251 /// Bit field extract
1252 bextr,
1253 /// Extract lowest set isolated bit
1254 /// Get mask up to lowest set bit
1255 /// Reset lowest set bit
1256 bls,
1257 /// Count the number of trailing zero bits
1258 tzcnt,
1259
1260 // BMI2
1261 /// Zero high bits starting with specified bit position
1262 bzhi,
1263 /// Parallel bits deposit
1264 pdep,
1265 /// Parallel bits extract
1266 pext,
1267
1268 // F16C
1269 /// Convert 16-bit floating-point values to single-precision floating-point values
1270 cvtph2,
1271 /// Convert single-precision floating-point values to 16-bit floating-point values
1272 cvtps2ph,
1273
1274 // FMA
1275 /// Fused multiply-add of packed single-precision floating-point values
1276 /// Fused multiply-add of scalar single-precision floating-point values
1277 /// Fused multiply-add of packed double-precision floating-point values
1278 /// Fused multiply-add of scalar double-precision floating-point values
1279 fmadd132,
1280 /// Fused multiply-add of packed single-precision floating-point values
1281 /// Fused multiply-add of scalar single-precision floating-point values
1282 /// Fused multiply-add of packed double-precision floating-point values
1283 /// Fused multiply-add of scalar double-precision floating-point values
1284 fmadd213,
1285 /// Fused multiply-add of packed single-precision floating-point values
1286 /// Fused multiply-add of scalar single-precision floating-point values
1287 /// Fused multiply-add of packed double-precision floating-point values
1288 /// Fused multiply-add of scalar double-precision floating-point values
1289 fmadd231,
1290
1291 // AVX2
1292 /// Permute packed doubleword elements
1293 /// Permute packed qword elements
1294 /// Permute double-precision floating-point elements
1295 /// Permute single-precision floating-point elements
1296 perm,
1297 /// Variable bit shift left logical
1298 sllv,
1299 /// Variable bit shift right arithmetic
1300 srav,
1301 /// Variable bit shift right logical
1302 srlv,
1303
1304 // ADX
1305 /// Unsigned integer addition of two operands with overflow flag
1306 ado,
1307
1308 // AESKLE
1309 /// Encode 128-bit key with key locker
1310 /// Encode 256-bit key with key locker
1311 encodekey,
1312 /// Load internal wrapping key with key locker
1313 loadiwkey,
1314
1315 /// A pseudo instruction that requires special lowering.
1316 /// This should be the only tag in this enum that doesn't
1317 /// directly correspond to one or more instruction mnemonics.
1318 pseudo,
1319 };
1320
1321 pub const FixedTag = struct { Fixes, Tag };
1322
1323 pub const Ops = enum(u8) {
1324 /// No data associated with this instruction (only mnemonic is used).
1325 none,
1326 /// Single register operand.
1327 /// Uses `r` payload.
1328 r,
1329 /// Register, register operands.
1330 /// Uses `rr` payload.
1331 rr,
1332 /// Register, register, register operands.
1333 /// Uses `rrr` payload.
1334 rrr,
1335 /// Register, register, register, register operands.
1336 /// Uses `rrrr` payload.
1337 rrrr,
1338 /// Register, register, register, immediate (byte) operands.
1339 /// Uses `rrri` payload.
1340 rrri,
1341 /// Register, register, immediate (sign-extended) operands.
1342 /// Uses `rri` payload.
1343 rri_s,
1344 /// Register, register, immediate (unsigned) operands.
1345 /// Uses `rri` payload.
1346 rri_u,
1347 /// Register, immediate (sign-extended) operands.
1348 /// Uses `ri` payload.
1349 ri_s,
1350 /// Register, immediate (unsigned) operands.
1351 /// Uses `ri` payload.
1352 ri_u,
1353 /// Register, 64-bit unsigned immediate operands.
1354 /// Uses `ri` payload with `i` index of extra data of type `Imm64`.
1355 ri_64,
1356 /// Immediate (sign-extended) operand.
1357 /// Uses `i` payload.
1358 i_s,
1359 /// Immediate (unsigned) operand.
1360 /// Uses `i` payload.
1361 i_u,
1362 /// Immediate (word), immediate (byte) operands.
1363 /// Uses `ii` payload.
1364 ii,
1365 /// Immediate (byte), register operands.
1366 /// Uses `ri` payload.
1367 ir,
1368 /// Register, memory operands.
1369 /// Uses `rx` payload with extra data of type `Memory`.
1370 rm,
1371 /// Register, memory, register operands.
1372 /// Uses `rrx` payload with extra data of type `Memory`.
1373 rmr,
1374 /// Register, memory, immediate (signed word) operands.
1375 /// Uses `rix` payload with extra data of type `Memory`.
1376 rmi,
1377 /// Register, memory, immediate (signed) operands.
1378 /// Uses `rx` payload with extra data of type `Imm32` followed by `Memory`.
1379 rmi_s,
1380 /// Register, memory, immediate (unsigned) operands.
1381 /// Uses `rx` payload with extra data of type `Imm32` followed by `Memory`.
1382 rmi_u,
1383 /// Register, register, memory.
1384 /// Uses `rrix` payload with extra data of type `Memory`.
1385 rrm,
1386 /// Register, register, memory, register.
1387 /// Uses `rrrx` payload with extra data of type `Memory`.
1388 rrmr,
1389 /// Register, register, memory, immediate (byte) operands.
1390 /// Uses `rrix` payload with extra data of type `Memory`.
1391 rrmi,
1392 /// Single memory operand.
1393 /// Uses `x` payload with extra data of type `Memory`.
1394 m,
1395 /// Memory, immediate (sign-extend) operands.
1396 /// Uses `x` payload with extra data of type `Imm32` followed by `Memory`.
1397 mi_s,
1398 /// Memory, immediate (unsigned) operands.
1399 /// Uses `x` payload with extra data of type `Imm32` followed by `Memory`.
1400 mi_u,
1401 /// Memory, register operands.
1402 /// Uses `rx` payload with extra data of type `Memory`.
1403 mr,
1404 /// Memory, register, register operands.
1405 /// Uses `rrx` payload with extra data of type `Memory`.
1406 mrr,
1407 /// Memory, register, immediate (signed word) operands.
1408 /// Uses `rix` payload with extra data of type `Memory`.
1409 mri,
1410 /// References another Mir instruction directly.
1411 /// Uses `inst` payload.
1412 inst,
1413 /// References a nav.
1414 /// Uses `nav` payload.
1415 nav,
1416 /// References an uav.
1417 /// Uses `uav` payload.
1418 uav,
1419 /// References a lazy symbol.
1420 /// Uses `lazy_sym` payload.
1421 lazy_sym,
1422 /// References an external symbol.
1423 /// Uses `extern_func` payload.
1424 extern_func,
1425
1426 // Pseudo instructions:
1427
1428 /// Conditional move if zero flag set and parity flag not set
1429 /// Clobbers the source operand!
1430 /// Uses `rr` payload.
1431 pseudo_cmov_z_and_np_rr,
1432 /// Conditional move if zero flag not set or parity flag set
1433 /// Uses `rr` payload.
1434 pseudo_cmov_nz_or_p_rr,
1435 /// Conditional move if zero flag not set or parity flag set
1436 /// Uses `rx` payload.
1437 pseudo_cmov_nz_or_p_rm,
1438 /// Set byte if zero flag set and parity flag not set
1439 /// Requires a scratch register!
1440 /// Uses `rr` payload.
1441 pseudo_set_z_and_np_r,
1442 /// Set byte if zero flag set and parity flag not set
1443 /// Requires a scratch register!
1444 /// Uses `rx` payload.
1445 pseudo_set_z_and_np_m,
1446 /// Set byte if zero flag not set or parity flag set
1447 /// Requires a scratch register!
1448 /// Uses `rr` payload.
1449 pseudo_set_nz_or_p_r,
1450 /// Set byte if zero flag not set or parity flag set
1451 /// Requires a scratch register!
1452 /// Uses `rx` payload.
1453 pseudo_set_nz_or_p_m,
1454 /// Jump if zero flag set and parity flag not set
1455 /// Uses `inst` payload.
1456 pseudo_j_z_and_np_inst,
1457 /// Jump if zero flag not set or parity flag set
1458 /// Uses `inst` payload.
1459 pseudo_j_nz_or_p_inst,
1460
1461 /// Probe alignment
1462 /// Uses `ri` payload.
1463 pseudo_probe_align_ri_s,
1464 /// Probe adjust unrolled
1465 /// Uses `ri` payload.
1466 pseudo_probe_adjust_unrolled_ri_s,
1467 /// Probe adjust setup
1468 /// Uses `rri` payload.
1469 pseudo_probe_adjust_setup_rri_s,
1470 /// Probe adjust loop
1471 /// Uses `rr` payload.
1472 pseudo_probe_adjust_loop_rr,
1473
1474 /// Push registers
1475 /// Uses `reg_list` payload.
1476 pseudo_push_reg_list,
1477 /// Pop registers
1478 /// Uses `reg_list` payload.
1479 pseudo_pop_reg_list,
1480
1481 /// Define cfa rule as offset from register.
1482 /// Uses `ri` payload.
1483 pseudo_cfi_def_cfa_ri_s,
1484 /// Modify cfa rule register.
1485 /// Uses `r` payload.
1486 pseudo_cfi_def_cfa_register_r,
1487 /// Modify cfa rule offset.
1488 /// Uses `i` payload.
1489 pseudo_cfi_def_cfa_offset_i_s,
1490 /// Offset cfa rule offset.
1491 /// Uses `i` payload.
1492 pseudo_cfi_adjust_cfa_offset_i_s,
1493 /// Define register rule as stored at offset from cfa.
1494 /// Uses `ri` payload.
1495 pseudo_cfi_offset_ri_s,
1496 /// Define register rule as offset from cfa.
1497 /// Uses `ri` payload.
1498 pseudo_cfi_val_offset_ri_s,
1499 /// Define register rule as stored at offset from cfa rule register.
1500 /// Uses `ri` payload.
1501 pseudo_cfi_rel_offset_ri_s,
1502 /// Define register rule as register.
1503 /// Uses `rr` payload.
1504 pseudo_cfi_register_rr,
1505 /// Define register rule from initial.
1506 /// Uses `r` payload.
1507 pseudo_cfi_restore_r,
1508 /// Define register rule as undefined.
1509 /// Uses `r` payload.
1510 pseudo_cfi_undefined_r,
1511 /// Define register rule as itself.
1512 /// Uses `r` payload.
1513 pseudo_cfi_same_value_r,
1514 /// Push cfi state.
1515 pseudo_cfi_remember_state_none,
1516 /// Pop cfi state.
1517 pseudo_cfi_restore_state_none,
1518 /// Raw cfi bytes.
1519 /// Uses `bytes` payload.
1520 pseudo_cfi_escape_bytes,
1521
1522 /// End of prologue
1523 /// Uses `none` payload.
1524 pseudo_dbg_prologue_end_none,
1525 /// Update debug line with is_stmt register set
1526 /// Uses `line_column` payload.
1527 pseudo_dbg_line_stmt_line_column,
1528 /// Update debug line with is_stmt register clear
1529 /// Uses `line_column` payload.
1530 pseudo_dbg_line_line_column,
1531 /// Start of epilogue
1532 /// Uses `none` payload.
1533 pseudo_dbg_epilogue_begin_none,
1534 /// Start of lexical block
1535 /// Uses `none` payload.
1536 pseudo_dbg_enter_block_none,
1537 /// End of lexical block
1538 /// Uses `none` payload.
1539 pseudo_dbg_leave_block_none,
1540 /// Start of inline function
1541 /// Uses `ip_index` payload.
1542 pseudo_dbg_enter_inline_func,
1543 /// End of inline function
1544 /// Uses `ip_index` payload.
1545 pseudo_dbg_leave_inline_func,
1546 /// Local argument.
1547 /// Uses `none` payload.
1548 pseudo_dbg_arg_none,
1549 /// Local argument.
1550 /// Uses `i` payload.
1551 pseudo_dbg_arg_i_s,
1552 /// Local argument.
1553 /// Uses `i` payload.
1554 pseudo_dbg_arg_i_u,
1555 /// Local argument.
1556 /// Uses `i64` payload.
1557 pseudo_dbg_arg_i_64,
1558 /// Local argument.
1559 /// Uses `ro` payload.
1560 pseudo_dbg_arg_ro,
1561 /// Local argument.
1562 /// Uses `fa` payload.
1563 pseudo_dbg_arg_fa,
1564 /// Local argument.
1565 /// Uses `x` payload with extra data of type `Memory`.
1566 pseudo_dbg_arg_m,
1567 /// Local argument.
1568 /// Uses `ip_index` payload.
1569 pseudo_dbg_arg_val,
1570 /// Remaining arguments are varargs.
1571 pseudo_dbg_var_args_none,
1572 /// Local variable.
1573 /// Uses `none` payload.
1574 pseudo_dbg_var_none,
1575 /// Local variable.
1576 /// Uses `i` payload.
1577 pseudo_dbg_var_i_s,
1578 /// Local variable.
1579 /// Uses `i` payload.
1580 pseudo_dbg_var_i_u,
1581 /// Local variable.
1582 /// Uses `i64` payload.
1583 pseudo_dbg_var_i_64,
1584 /// Local variable.
1585 /// Uses `ro` payload.
1586 pseudo_dbg_var_ro,
1587 /// Local variable.
1588 /// Uses `fa` payload.
1589 pseudo_dbg_var_fa,
1590 /// Local variable.
1591 /// Uses `x` payload with extra data of type `Memory`.
1592 pseudo_dbg_var_m,
1593 /// Local variable.
1594 /// Uses `ip_index` payload.
1595 pseudo_dbg_var_val,
1596
1597 /// Tombstone
1598 /// Emitter should skip this instruction.
1599 pseudo_dead_none,
1600 };
1601
1602 pub const Data = union {
1603 none: struct {
1604 fixes: Fixes = ._,
1605 },
1606 /// References another Mir instruction.
1607 inst: struct {
1608 fixes: Fixes = ._,
1609 inst: Index,
1610 },
1611 /// A 32-bit immediate value.
1612 i64: u64,
1613 i: struct {
1614 fixes: Fixes = ._,
1615 i: u32,
1616 },
1617 ii: struct {
1618 fixes: Fixes = ._,
1619 i1: u16,
1620 i2: u8,
1621 },
1622 r: struct {
1623 fixes: Fixes = ._,
1624 r1: Register,
1625 },
1626 rr: struct {
1627 fixes: Fixes = ._,
1628 r1: Register,
1629 r2: Register,
1630 },
1631 rrr: struct {
1632 fixes: Fixes = ._,
1633 r1: Register,
1634 r2: Register,
1635 r3: Register,
1636 },
1637 rrrr: struct {
1638 fixes: Fixes = ._,
1639 r1: Register,
1640 r2: Register,
1641 r3: Register,
1642 r4: Register,
1643 },
1644 rrri: struct {
1645 fixes: Fixes = ._,
1646 r1: Register,
1647 r2: Register,
1648 r3: Register,
1649 i: u8,
1650 },
1651 rri: struct {
1652 fixes: Fixes = ._,
1653 r1: Register,
1654 r2: Register,
1655 i: u32,
1656 },
1657 /// Register, immediate.
1658 ri: struct {
1659 fixes: Fixes = ._,
1660 r1: Register,
1661 i: u32,
1662 },
1663 /// Register, followed by custom payload found in extra.
1664 rx: struct {
1665 fixes: Fixes = ._,
1666 r1: Register,
1667 payload: u32,
1668 },
1669 /// Register, register, followed by Custom payload found in extra.
1670 rrx: struct {
1671 fixes: Fixes = ._,
1672 r1: Register,
1673 r2: Register,
1674 payload: u32,
1675 },
1676 /// Register, register, register, followed by Custom payload found in extra.
1677 rrrx: struct {
1678 fixes: Fixes = ._,
1679 r1: Register,
1680 r2: Register,
1681 r3: Register,
1682 payload: u32,
1683 },
1684 /// Register, byte immediate, followed by Custom payload found in extra.
1685 rix: struct {
1686 fixes: Fixes = ._,
1687 r1: Register,
1688 i: i16,
1689 payload: u32,
1690 },
1691 /// Register, register, byte immediate, followed by Custom payload found in extra.
1692 rrix: struct {
1693 fixes: Fixes = ._,
1694 r1: Register,
1695 r2: Register,
1696 i: u8,
1697 payload: u32,
1698 },
1699 /// Custom payload found in extra.
1700 x: struct {
1701 fixes: Fixes = ._,
1702 payload: u32,
1703 },
1704 bytes: struct {
1705 payload: u32,
1706 len: u32,
1707
1708 pub fn get(bytes: @This(), mir: Mir) []const u8 {
1709 return std.mem.sliceAsBytes(mir.extra[bytes.payload..])[0..bytes.len];
1710 }
1711 },
1712 fa: bits.FrameAddr,
1713 ro: bits.RegisterOffset,
1714 nav: bits.NavOffset,
1715 uav: InternPool.Key.Ptr.BaseAddr.Uav,
1716 lazy_sym: link.File.LazySymbol,
1717 extern_func: Mir.NullTerminatedString,
1718 /// Debug line and column position
1719 line_column: struct {
1720 line: u32,
1721 column: u32,
1722 },
1723 ip_index: InternPool.Index,
1724 /// Register list
1725 reg_list: RegisterList,
1726 };
1727
1728 comptime {
1729 if (!std.debug.runtime_safety) {
1730 // Make sure we don't accidentally make instructions bigger than expected.
1731 // Note that in safety builds, Zig is allowed to insert a secret field for safety checks.
1732 assert(@sizeOf(Data) == 8);
1733 }
1734 const Mnemonic = @import("Encoding.zig").Mnemonic;
1735 if (@typeInfo(Mnemonic).@"enum".field_names.len != 978 or
1736 @typeInfo(Fixes).@"enum".field_names.len != 231 or
1737 @typeInfo(Tag).@"enum".field_names.len != 251)
1738 {
1739 const cond_src = (struct {
1740 fn src() std.lang.SourceLocation {
1741 return @src();
1742 }
1743 }).src();
1744 @setEvalBranchQuota(2_000_000);
1745 for (@typeInfo(Mnemonic).@"enum".field_names) |mnemonic_name| {
1746 if (mnemonic_name[0] == '.') continue;
1747 for (@typeInfo(Fixes).@"enum".field_names) |fixes_name| {
1748 const pattern = fixes_name[if (std.mem.findScalar(u8, fixes_name, ' ')) |index| index + " ".len else 0..];
1749 const wildcard_index = std.mem.findScalar(u8, pattern, '_').?;
1750 const mnem_prefix = pattern[0..wildcard_index];
1751 const mnem_suffix = pattern[wildcard_index + "_".len ..];
1752 if (!std.mem.startsWith(u8, mnemonic_name, mnem_prefix)) continue;
1753 if (!std.mem.endsWith(u8, mnemonic_name, mnem_suffix)) continue;
1754 if (@hasField(
1755 Tag,
1756 mnemonic_name[mnem_prefix.len .. mnemonic_name.len - mnem_suffix.len],
1757 )) break;
1758 } else @compileError("'" ++ mnemonic_name ++ "' is not encodable in Mir");
1759 }
1760 @compileError(std.fmt.comptimePrint(
1761 \\All mnemonics are encodable in Mir! You may now change the condition at {s}:{d} to:
1762 \\if (@typeInfo(Mnemonic).@"enum".field_names.len != {d} or
1763 \\ @typeInfo(Fixes).@"enum".field_names.len != {d} or
1764 \\ @typeInfo(Tag).@"enum".field_names.len != {d})
1765 , .{
1766 cond_src.file,
1767 cond_src.line - 6,
1768 @typeInfo(Mnemonic).@"enum".field_names.len,
1769 @typeInfo(Fixes).@"enum".field_names.len,
1770 @typeInfo(Tag).@"enum".field_names.len,
1771 }));
1772 }
1773 }
1774};
1775
1776/// Used in conjunction with payload to transfer a list of used registers in a compact manner.
1777pub const RegisterList = struct {
1778 bitset: BitSet,
1779
1780 const BitSet = std.bit_set.Integer(32);
1781 const Self = @This();
1782
1783 pub const empty: RegisterList = .{ .bitset = .empty };
1784
1785 fn getIndexForReg(registers: []const Register, reg: Register) BitSet.MaskInt {
1786 for (registers, 0..) |cpreg, i| {
1787 if (reg.id() == cpreg.id()) return @intCast(i);
1788 }
1789 unreachable; // register not in input register list!
1790 }
1791
1792 pub fn push(self: *Self, registers: []const Register, reg: Register) void {
1793 const index = getIndexForReg(registers, reg);
1794 self.bitset.set(index);
1795 }
1796
1797 pub fn isSet(self: Self, registers: []const Register, reg: Register) bool {
1798 const index = getIndexForReg(registers, reg);
1799 return self.bitset.isSet(index);
1800 }
1801
1802 pub fn iterator(self: Self, comptime options: std.bit_set.IteratorOptions) BitSet.Iterator(options) {
1803 return self.bitset.iterator(options);
1804 }
1805
1806 pub fn count(self: Self) i32 {
1807 return @intCast(self.bitset.count());
1808 }
1809
1810 pub fn size(self: Self, target: *const std.Target) i32 {
1811 return @intCast(self.bitset.count() * @as(u4, switch (target.cpu.arch) {
1812 else => unreachable,
1813 .x86 => 4,
1814 .x86_64 => 8,
1815 }));
1816 }
1817};
1818
1819pub const NullTerminatedString = enum(u32) {
1820 none = std.math.maxInt(u32),
1821 _,
1822
1823 pub fn toSlice(nts: NullTerminatedString, mir: *const Mir) ?[:0]const u8 {
1824 if (nts == .none) return null;
1825 const string_bytes = mir.string_bytes[@backingInt(nts)..];
1826 return string_bytes[0..std.mem.findScalar(u8, string_bytes, 0).? :0];
1827 }
1828};
1829
1830pub const Local = struct {
1831 name: NullTerminatedString,
1832 type: InternPool.Index,
1833};
1834
1835pub const Imm32 = struct {
1836 imm: u32,
1837};
1838
1839pub const Imm64 = struct {
1840 msb: u32,
1841 lsb: u32,
1842
1843 pub fn encode(v: u64) Imm64 {
1844 return .{
1845 .msb = @truncate(v >> 32),
1846 .lsb = @truncate(v),
1847 };
1848 }
1849
1850 pub fn decode(imm: Imm64) u64 {
1851 var res: u64 = 0;
1852 res |= @as(u64, @intCast(imm.msb)) << 32;
1853 res |= @as(u64, @intCast(imm.lsb));
1854 return res;
1855 }
1856};
1857
1858pub const Memory = struct {
1859 info: Info,
1860 base: u32,
1861 off: u32,
1862 extra: u32,
1863
1864 pub const Info = packed struct(u32) {
1865 base: @typeInfo(bits.Memory.Base).@"union".tag_type.?,
1866 mod: @typeInfo(bits.Memory.Mod).@"union".tag_type.?,
1867 size: bits.Memory.Size,
1868 index: Register,
1869 scale: bits.Memory.Scale,
1870 _: u13 = undefined,
1871 };
1872
1873 pub fn encode(mem: bits.Memory) Memory {
1874 return .{
1875 .info = .{
1876 .base = mem.base,
1877 .mod = mem.mod,
1878 .size = switch (mem.mod) {
1879 .rm => |rm| rm.size,
1880 .off => undefined,
1881 },
1882 .index = switch (mem.mod) {
1883 .rm => |rm| rm.index,
1884 .off => undefined,
1885 },
1886 .scale = switch (mem.mod) {
1887 .rm => |rm| rm.scale,
1888 .off => undefined,
1889 },
1890 },
1891 .base = switch (mem.base) {
1892 .none, .table => undefined,
1893 .reg => |reg| @backingInt(reg),
1894 .frame => |frame_index| @backingInt(frame_index),
1895 .rip_inst => |inst_index| inst_index,
1896 .nav => |nav| @backingInt(nav),
1897 .uav => |uav| @backingInt(uav.val),
1898 .lazy_sym => |lazy_sym| @backingInt(lazy_sym.ty),
1899 .extern_func => |extern_func| @backingInt(extern_func),
1900 },
1901 .off = switch (mem.mod) {
1902 .rm => |rm| @bitCast(rm.disp),
1903 .off => |off| @truncate(off),
1904 },
1905 .extra = switch (mem.mod) {
1906 .rm => switch (mem.base) {
1907 else => undefined,
1908 .uav => |uav| @backingInt(uav.orig_ty),
1909 .lazy_sym => |lazy_sym| @backingInt(lazy_sym.kind),
1910 },
1911 .off => switch (mem.base) {
1912 .reg => @intCast(mem.mod.off >> 32),
1913 else => unreachable,
1914 },
1915 },
1916 };
1917 }
1918
1919 pub fn decode(mem: Memory) encoder.Instruction.Memory {
1920 switch (mem.info.mod) {
1921 .rm => {
1922 if (mem.info.base == .reg and @as(Register, @fromBackingInt(@intCast(mem.base))) == .rip) {
1923 assert(mem.info.index == .none and mem.info.scale == .@"1");
1924 return encoder.Instruction.Memory.initRip(mem.info.size, @bitCast(mem.off));
1925 }
1926 return encoder.Instruction.Memory.initSib(mem.info.size, .{
1927 .disp = @bitCast(mem.off),
1928 .base = switch (mem.info.base) {
1929 .none => .none,
1930 .reg => .{ .reg = @fromBackingInt(@intCast(mem.base)) },
1931 .frame => .{ .frame = @fromBackingInt(@intCast(mem.base)) },
1932 .table => .table,
1933 .rip_inst => .{ .rip_inst = mem.base },
1934 .nav => .{ .nav = @fromBackingInt(@intCast(mem.base)) },
1935 .uav => .{ .uav = .{ .val = @fromBackingInt(@intCast(mem.base)), .orig_ty = @fromBackingInt(@intCast(mem.extra)) } },
1936 .lazy_sym => .{ .lazy_sym = .{ .kind = @fromBackingInt(@intCast(mem.extra)), .ty = @fromBackingInt(@intCast(mem.base)) } },
1937 .extern_func => .{ .extern_func = @fromBackingInt(@intCast(mem.base)) },
1938 },
1939 .scale_index = switch (mem.info.index) {
1940 .none => null,
1941 else => |index| .{ .scale = switch (mem.info.scale) {
1942 inline else => |scale| comptime std.fmt.parseInt(
1943 u4,
1944 @tagName(scale),
1945 10,
1946 ) catch unreachable,
1947 }, .index = index },
1948 },
1949 });
1950 },
1951 .off => {
1952 assert(mem.info.base == .reg);
1953 return encoder.Instruction.Memory.initMoffs(
1954 @fromBackingInt(@intCast(mem.base)),
1955 @as(u64, mem.extra) << 32 | mem.off,
1956 );
1957 },
1958 }
1959 }
1960};
1961
1962pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {
1963 mir.instructions.deinit(gpa);
1964 gpa.free(mir.extra);
1965 gpa.free(mir.string_bytes);
1966 gpa.free(mir.locals);
1967 gpa.free(mir.table);
1968 gpa.free(mir.memoized_encodings);
1969 mir.frame_locs.deinit(gpa);
1970 mir.* = undefined;
1971}
1972
1973pub fn emit(
1974 mir: Mir,
1975 lf: *link.File,
1976 pt: Zcu.PerThread,
1977 func_index: InternPool.Index,
1978 atom_id: link.File.AtomId,
1979 w: *std.Io.Writer,
1980 debug_output: link.File.DebugInfoOutput,
1981) codegen.Error!void {
1982 const zcu = pt.zcu;
1983 const comp = zcu.comp;
1984 const gpa = comp.gpa;
1985 const func = zcu.funcInfo(func_index);
1986 const fn_info = zcu.typeToFunc(.fromInterned(func.ty)).?;
1987 const nav = func.owner_nav;
1988 const mod = zcu.navFileScope(nav).mod.?;
1989 var e: Emit = .{
1990 .lower = .{
1991 .target = &mod.resolved_target.result,
1992 .allocator = gpa,
1993 .mir = mir,
1994 .cc = fn_info.cc,
1995 .src_loc = zcu.navSrcLoc(nav),
1996 },
1997 .bin_file = lf,
1998 .pt = pt,
1999 .pic = mod.pic,
2000 .atom_id = atom_id,
2001 .debug_output = debug_output,
2002 .w = w,
2003
2004 .prev_di_loc = .{
2005 .line = func.lbrace_line,
2006 .column = func.lbrace_column,
2007 .is_stmt = switch (debug_output) {
2008 .dwarf => |dwarf| dwarf.dwarf.debug_line.header.default_is_stmt,
2009 .none => undefined,
2010 },
2011 },
2012 .prev_di_pc = 0,
2013
2014 .code_offset_mapping = .empty,
2015 .relocs = .empty,
2016 .table_relocs = .empty,
2017 };
2018 defer e.deinit();
2019 e.emitMir() catch |err| switch (err) {
2020 error.LowerFail, error.EmitFail => return zcu.codegenFailMsg(nav, e.lower.err_msg.?),
2021 error.InvalidInstruction, error.CannotEncode => return zcu.codegenFail(nav, "emit MIR failed: {s} (Zig compiler bug)", .{@errorName(err)}),
2022 else => return zcu.codegenFail(nav, "emit MIR failed: {s}", .{@errorName(err)}),
2023 };
2024}
2025
2026pub fn emitLazy(
2027 mir: Mir,
2028 lf: *link.File,
2029 pt: Zcu.PerThread,
2030 lazy_sym: link.File.LazySymbol,
2031 atom_id: link.File.AtomId,
2032 w: *std.Io.Writer,
2033 debug_output: link.File.DebugInfoOutput,
2034) codegen.Error!void {
2035 const zcu = pt.zcu;
2036 const comp = zcu.comp;
2037 const gpa = comp.gpa;
2038 const mod = comp.root_mod;
2039 var e: Emit = .{
2040 .lower = .{
2041 .target = &mod.resolved_target.result,
2042 .allocator = gpa,
2043 .mir = mir,
2044 .cc = .auto,
2045 .src_loc = Zcu.Type.fromInterned(lazy_sym.ty).srcLocOrNull(zcu) orelse .unneeded,
2046 },
2047 .bin_file = lf,
2048 .pt = pt,
2049 .pic = mod.pic,
2050 .atom_id = atom_id,
2051 .debug_output = debug_output,
2052 .w = w,
2053
2054 .prev_di_loc = undefined,
2055 .prev_di_pc = undefined,
2056
2057 .code_offset_mapping = .empty,
2058 .relocs = .empty,
2059 .table_relocs = .empty,
2060 };
2061 defer e.deinit();
2062 e.emitMir() catch |err| switch (err) {
2063 error.LowerFail, error.EmitFail => return zcu.codegenFailTypeMsg(lazy_sym.ty, e.lower.err_msg.?),
2064 error.InvalidInstruction, error.CannotEncode => return zcu.codegenFailType(lazy_sym.ty, "emit MIR failed: {s} (Zig compiler bug)", .{@errorName(err)}),
2065 else => return zcu.codegenFailType(lazy_sym.ty, "emit MIR failed: {s}", .{@errorName(err)}),
2066 };
2067}
2068
2069pub fn extraData(mir: Mir, comptime T: type, index: u32) struct { data: T, end: u32 } {
2070 const info = @typeInfo(T).@"struct";
2071 var i: u32 = index;
2072 var result: T = undefined;
2073 inline for (info.field_names, info.field_types) |field_name, field_type| {
2074 @field(result, field_name) = switch (field_type) {
2075 u32 => mir.extra[i],
2076 i32, Memory.Info => @bitCast(mir.extra[i]),
2077 bits.FrameIndex => @fromBackingInt(@intCast(mir.extra[i])),
2078 else => @compileError("bad field type: " ++ field_name ++ ": " ++ @typeName(field_type)),
2079 };
2080 i += 1;
2081 }
2082 return .{
2083 .data = result,
2084 .end = i,
2085 };
2086}
2087
2088pub const FrameLoc = struct {
2089 base: Register,
2090 disp: i32,
2091};
2092
2093pub fn resolveFrameAddr(mir: Mir, frame_addr: bits.FrameAddr) bits.RegisterOffset {
2094 const frame_loc = mir.frame_locs.get(@backingInt(frame_addr.index));
2095 return .{ .reg = frame_loc.base, .off = frame_loc.disp + frame_addr.off };
2096}
2097
2098pub fn resolveMemoryExtra(mir: Mir, payload: u32) Memory {
2099 const mem = mir.extraData(Mir.Memory, payload).data;
2100 return switch (mem.info.base) {
2101 .none, .reg, .table, .rip_inst, .nav, .uav, .lazy_sym, .extern_func => mem,
2102 .frame => if (mir.frame_locs.len > 0) .{
2103 .info = .{
2104 .base = .reg,
2105 .mod = mem.info.mod,
2106 .size = mem.info.size,
2107 .index = mem.info.index,
2108 .scale = mem.info.scale,
2109 },
2110 .base = @backingInt(mir.frame_locs.items(.base)[mem.base]),
2111 .off = @bitCast(mir.frame_locs.items(.disp)[mem.base] + @as(i32, @bitCast(mem.off))),
2112 .extra = mem.extra,
2113 } else mem,
2114 };
2115}
2116
2117const assert = std.debug.assert;
2118const bits = @import("bits.zig");
2119const builtin = @import("builtin");
2120const encoder = @import("encoder.zig");
2121const std = @import("std");
2122
2123const InternPool = @import("../../InternPool.zig");
2124const Mir = @This();
2125const Register = bits.Register;
2126const Emit = @import("Emit.zig");
2127const codegen = @import("../../codegen.zig");
2128const link = @import("../../link.zig");
2129const Zcu = @import("../../Zcu.zig");