[Info-vax] debugging stack corruption on Macro64?

[John E. Malmberg]

On 3/8/2015 11:26 PM, John E. Malmberg wrote:
>
> The R17 value appears to be quadword aligned when the corruption occurs.

I am now reproducing the stack corruption with this program once I made 
a change to save some registers.  The test macro code does not use the 
registers, the actual program does.


EAGLE> type [.vms]test.c,test2.m64

#define __NEW_STARLET 1
#include <psigdef.h>
#include <stdio.h>
#include <builtins.h>
#pragma message disable badptrarith

extern void ffi_call_osf(void *stack, void *frame, unsigned flags,
                          void *raddr, void (*fn)(void), void *closure);


int main(int argc, char **argv) {

unsigned long * argp;
void * frame;
unsigned flags;
int ret_value;
long long * stack;
AIDEF *vms_ai;
unsigned long long *vms_ai_llong;


     argp = frame = __ALLOCA(8 * (6 + 4));
     frame += (8 * 6);
     vms_ai = (AIDEF *) frame;
     vms_ai_llong = (unsigned long long *) frame;
     vms_ai_llong[0] = 0;
     vms_ai->ai$b_arg_count = 1;

     flags = 1;
     stack = (long long *) argp;
     stack[0] = (long long)"test string";

     ffi_call_osf(argp, frame, flags, &ret_value, (void (*)(void))puts, 
NULL);

}

VMS_ROOT:[libffi.vms]test2.m64

         .TITLE "VMS"

         .PSECT MY_CODE,EXE,NOMIX
         .align long

         $ROUTINE ffi_call_osf,-
                  SAVED_REGS=<R2,R3,R4,FP>,-
                  KIND=STACK,-
                  STANDARD_PROLOGUE=FALSE
         $CODE_SECTION

         lda     sp,-24(sp)
         stq     r27,(sp)
         stq     r26,8(sp)
         stq     fp,16(sp)
; step/inst to here ok.
         mov     sp,fp
; step/inst
; DEBUG-I-INCMPSTACK, WARNING: stack corrupted; session integrity not 
guaranteed.
; fp and sp now contain 07ad8b9a8

         ldq     r25, 0(r17)     ; VMS AI from base of frame
         and     r25, #255, r22  ; Get the argcount
         mov     r16, r23        ; argp to fake stackpointer
         stq     r26, 0(r17)     ; Return Address to frame[0] (64bit)
         stq     fp, 8(r17)      ; FP to frame[1] (64 bit)
         $end_prologue

         stq     r18, 16(r17)
         stq     r19, 24(r17)
         mov     r20, R27
         ldq     r16, 0(r23)
         ldt     F16, 0(R23)
         ldt     F17, 8(R23)
         ldq     R17, 8(R23)
         ldt     F18, 16(R23)
         ldq     R18, 16(R23)
         ldt     F19, 24(R23)
         ldq     R19, 24(R23)
         ldt     F20, 32(R23)
         ldq     R20, 32(R23)
         ldt     F21, 40(R23)
         ldq     R21, 40(R23)
         ldq     R26, 8(R27)
         jsr     R26, R26
         $return
         $END_ROUTINE ffi_call_osf

EAGLE> show sym cc
   CC == 
"CC/STANDARD=(RELAXED,ISOC94)/ACCEPT=(NOVAXC,RESTR,C99)/LIST/SHOW=(EXPAN,INCLU)/NAMES=(AS_IS,SHORT)/MAIN=POSIX_EXIT/FLOAT=IEEE/IEEE_MODE=FAST/NESTED=NONE/DEBUG/NOOPT"

EAGLE> CC test.c
EAGLE> macro/alpha/debug/name=as_is [.vms]test2.m64
EAGLE> link/exe=test2.exe test.obj,test2.obj/debug