[syslinux] com32: custom int3 handler

[Jeff Kalikstein]

Thanks, Peter-

After I sent the message, I discovered my mistake, and
am now able to call my int3 handler.  However, I am
not able to return correctly after the interrupt. 
Based on the x86 documentation, I suspect that the
problem is with adjusting the sp in the handler.  Do
you have any gcc tricks that can help?

Also, what do you think about adding com32 API
functions to install and uninstall 32-bit handlers? 
If you like the idea, I'd be happy to write a patch.

My current code follows (I now see "int3!!!", but not
"DONE"):


void int3_handler()
{
        __asm__("cli");
        put_str("int3!!!");
        __asm__("sti");
        __asm__("iret");
}

void init_handlers()
{
        struct {
                unsigned long limit     : 16;
                unsigned long base      : 32;
        } __attribute__((packed)) idtr;

        struct {
                unsigned long off_low   : 16;
                unsigned long cs        : 16;
                unsigned long reserved  : 8;
                unsigned long type      : 4;
                unsigned long zero      : 1;
                unsigned long dpl       : 2;
                unsigned long present   : 1;
                unsigned long off_high  : 16;
        } __attribute__((packed)) *idt;

        __asm__("sidt (%0)" :: "r"(&idtr));

        idt = (void*)idtr.base;

        unsigned short cs;
        idt[3].off_low = (unsigned long)int3_handler;
        idt[3].off_high = (unsigned long)int3_handler
>> 16;
        __asm__("mov %%cs, %%ax\n mov %%ax, (%0)" ::
"r"(&cs) : "%ax");
        idt[3].cs = cs;
        idt[3].type = 0xe;      // 32-bit trap gate
        idt[3].dpl = 0x3;
        idt[3].present = 1;

        put_str("about to try int3");
        __asm__("int3");
        put_str("DONE!");
        anykey();
}




--- "H. Peter Anvin" <hpa at zytor.com> wrote:

> H. Peter Anvin wrote:
> > Jeff Kalikstein wrote:
> > 
> >>
> >>         __asm__("sidt (%0)" :: "r"(&idtr));
> >>
> >>         unsigned long *idt = (void*)idtr.base;
> >>         idt[3] = int3_handler;
> >>
> > 
> > Check the format of an IDT descriptor; this isn't
> it.
> > 
> 
> Here is a function which should make a valid IDT
> entry.  Note that it's 
> 64 bits long, not 32.  "disable_int" controls if you
> want it to disable
> interrupts when the trap is taken or not.
> 
> #include <stdint.h>
> 
> static inline uint64_t
> make_idt_entry(uintptr_t handler, int disable_int)
> {
> 	uint64_t v;
> 	uint16_t cs;
> 
> 	asm("movw %%cs,%0" : "=rm" (cs));
> 
> 	v  = handler & 0xffff;
> 	v |= (uint64_t)cs << 16;
> 	v |= (uint64_t)(disable_int ? 0x8E00 : 0x8F00) <<
> 32;
> 	v |= (uint64_t)(handler & 0xffff0000) << 32;
> 
> 	return v;
> }
>