We start our survey of historical stack limit checking functions on Windows with the 80386 family of processors. This function has actually changed form over the years, so we’ll start with the “original flavor”.<\/p>\n
Originally, the A function with a large stack frame would go something like<\/p>\n This goes into the competition for “wackiest x86-32 calling convention.”\u00b9<\/p>\n Next time, we’ll look at how stack probing happens on MIPS, which has its own quirks, but nothing as crazy as this.<\/p>\n Bonus chatter<\/b>: The strange calling convention dates back to the 16-bit 8086. And back then, there were two versions of the One of the weirdest calling conventions you’ll see.<\/p>\n","protected":false},"author":1069,"featured_media":111744,"comment_status":"open","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"_acf_changed":false,"footnotes":""},"categories":[1],"tags":[25],"class_list":["post-112136","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-oldnewthing","tag-code"],"acf":[],"blog_post_summary":" One of the weirdest calling conventions you’ll see.<\/p>\n","_links":{"self":[{"href":"https:\/\/devblogs.microsoft.com\/oldnewthing\/wp-json\/wp\/v2\/posts\/112136","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/devblogs.microsoft.com\/oldnewthing\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/devblogs.microsoft.com\/oldnewthing\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/devblogs.microsoft.com\/oldnewthing\/wp-json\/wp\/v2\/users\/1069"}],"replies":[{"embeddable":true,"href":"https:\/\/devblogs.microsoft.com\/oldnewthing\/wp-json\/wp\/v2\/comments?post=112136"}],"version-history":[{"count":1,"href":"https:\/\/devblogs.microsoft.com\/oldnewthing\/wp-json\/wp\/v2\/posts\/112136\/revisions"}],"predecessor-version":[{"id":112162,"href":"https:\/\/devblogs.microsoft.com\/oldnewthing\/wp-json\/wp\/v2\/posts\/112136\/revisions\/112162"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/devblogs.microsoft.com\/oldnewthing\/wp-json\/wp\/v2\/media\/111744"}],"wp:attachment":[{"href":"https:\/\/devblogs.microsoft.com\/oldnewthing\/wp-json\/wp\/v2\/media?parent=112136"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/devblogs.microsoft.com\/oldnewthing\/wp-json\/wp\/v2\/categories?post=112136"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/devblogs.microsoft.com\/oldnewthing\/wp-json\/wp\/v2\/tags?post=112136"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}_chkstk<\/code> function was called by putting the desired number of bytes in the eax<\/code> register and calling the _chkstk<\/code> function. The function touched each page of the stack, adjusted the stack pointer, and then returned with the adjusted stack pointer<\/i>. This is an unusual calling convention since it is neither caller clean, nor is it callee clean. It’s callee-dirty! The function returns with more<\/i> stack than it started.<\/p>\n_chkstk:\r\n push ecx ; preserve register\r\n\r\n ; calculate the stack pointer of the caller\r\n mov ecx, esp\r\n add ecx, 8 ; 4 bytes were auto-pushed for the return address,\r\n ; we pushed 4 bytes for the ecx\r\n\r\ntouch:\r\n cmp eax, PAGE_SIZE ; less than a page to go?\r\n jb finalpage ; do the last page and finish\r\n sub ecx, PAGE_SIZE ; allocate a page from our pretend stack pointer\r\n or dword ptr [ecx], 0 ; touch the memory\r\n sub eax, PAGE_SIZE ; did a page\r\n jmp touch ; go back and do some more\r\n\r\nfinalpage:\r\n sub ecx, eax ; allocate the leftovers from our pretend stack pointer\r\n or dword ptr [ecx], 0 ; touch the memory\r\n mov eax, esp ; remember original stack pointer\r\n mov esp, ecx ; move the real stack to match our pretend stack\r\n mov ecx, [eax] ; recover original ecx\r\n mov eax, 4[eax] ; recover return address\r\n jmp eax ; \"return\" to caller\r\n<\/pre>\n
function:\r\n push ebp ; link into frame chain\r\n mov ebp, esp\r\n push ebx ; save non-volatile register\r\n push esi\r\n push edi\r\n mov eax, 17320 ; large stack frame\r\n call _chkstk ; allocate it from our stack safely\r\n ; behaves like \"sub esp, eax\"\r\n<\/pre>\n
chkstk<\/code> function, depending on whether you were calling it far or near.<\/p>\n; frame size in ax\r\n\r\nchkstk:\r\n#if NEAR\r\n pop bx ; pop 16-bit return address\r\n#else \/\/ FAR\r\n pop bx ; pop 32-bit return address\r\n pop dx\r\n#endif\r\n\r\n inc ax\r\n and al, 0xFE ; round up to even\r\n\r\n sub ax, sp ; check for stack overflow\r\n jae overflow ; Y: overflow\r\n neg ax ; ax = new stack pointer\r\n\r\n cmp ax, ss:[pStackTop]\r\n ja overflow ; stack mysteriously too high\r\n\r\n cmp ax, ss:[pStackMin] ; new stack limit?\r\n jbe nochange\r\n mov ss:[pStackMin], ax ; update stack limit\r\nnochange:\r\n\r\n mov sp, ax ; update the stack pointer\r\n\r\n#if NEAR\r\n jmp bx ; \"return\" to caller\r\n#else \/\/ FAR\r\n push dx ; restore return address\r\n push bx\r\n retf ; return to caller\r\n#endif\r\n<\/pre>\n","protected":false},"excerpt":{"rendered":"