From 491f271a9b2a0665c656dca006eff1db4b398c56 Mon Sep 17 00:00:00 2001 From: "N. Brouard" Date: Tue, 17 Jun 2003 13:12:43 +0000 Subject: [PATCH] * imach.c (Repository): Check when date of death was earlier that current date of interview. It may happen when the death was just prior to the death. In this case, dh was negative and likelihood was wrong (infinity). We still send an "Error" but patch by assuming that the date of death was just one stepm after the interview. (Repository): Because some people have very long ID (first column) we changed int to long in num[] and we added a new lvector for memory allocation. But we also truncated to 8 characters (left truncation) (Repository): No more line truncation errors. --- src/imach.c | 434 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 332 insertions(+), 102 deletions(-) diff --git a/src/imach.c b/src/imach.c index 7a88e88..f9efc13 100644 --- a/src/imach.c +++ b/src/imach.c @@ -1,6 +1,13 @@ /* $Id$ $State$ $Log$ + Revision 1.84 2003/06/13 21:44:43 brouard + * imach.c (Repository): Replace "freqsummary" at a correct + place. It differs from routine "prevalence" which may be called + many times. Probs is memory consuming and must be used with + parcimony. + Version 0.95a2 (should output exactly the same maximization than 0.8a2) + Revision 1.83 2003/06/10 13:39:11 lievre *** empty log message *** @@ -105,9 +112,9 @@ #define MAXLINE 256 #define GNUPLOTPROGRAM "gnuplot" /*#define GNUPLOTPROGRAM "..\\gp37mgw\\wgnuplot"*/ -#define FILENAMELENGTH 80 +#define FILENAMELENGTH 132 /*#define DEBUG*/ -#define windows +/*#define windows*/ #define GLOCK_ERROR_NOPATH -1 /* empty path */ #define GLOCK_ERROR_GETCWD -2 /* cannot get cwd */ @@ -122,12 +129,12 @@ #define YEARM 12. /* Number of months per year */ #define AGESUP 130 #define AGEBASE 40 -#ifdef windows -#define DIRSEPARATOR '\\' -#define ODIRSEPARATOR '/' -#else +#ifdef unix #define DIRSEPARATOR '/' #define ODIRSEPARATOR '\\' +#else +#define DIRSEPARATOR '\\' +#define ODIRSEPARATOR '/' #endif /* $Id$ */ @@ -157,6 +164,12 @@ double **oldm, **newm, **savm; /* Working pointers to matrices */ double **oldms, **newms, **savms; /* Fixed working pointers to matrices */ FILE *fic,*ficpar, *ficparo,*ficres, *ficrespl, *ficrespij, *ficrest,*ficresf,*ficrespop; FILE *ficlog, *ficrespow; +int globpr; /* Global variable for printing or not */ +double fretone; /* Only one call to likelihood */ +long ipmx; /* Number of contributions */ +double sw; /* Sum of weights */ +char fileresilk[FILENAMELENGTH]; /* File of individual contributions to the likelihood */ +FILE *ficresilk; FILE *ficgp,*ficresprob,*ficpop, *ficresprobcov, *ficresprobcor; FILE *ficresprobmorprev; FILE *fichtm; /* Html File */ @@ -214,7 +227,8 @@ int estepm; /* Estepm, step in month to interpolate survival function in order to approximate Life Expectancy*/ int m,nb; -int *num, firstpass=0, lastpass=4,*cod, *ncodemax, *Tage; +long *num; +int firstpass=0, lastpass=4,*cod, *ncodemax, *Tage; double **agev,*moisnais, *annais, *moisdc, *andc,**mint, **anint; double **pmmij, ***probs; double dateintmean=0; @@ -254,11 +268,12 @@ static int split( char *path, char *dirc, char *name, char *ext, char *finame ) dirc[l1-l2] = 0; /* add zero */ } l1 = strlen( dirc ); /* length of directory */ -#ifdef windows + /*#ifdef windows if ( dirc[l1-1] != '\\' ) { dirc[l1] = '\\'; dirc[l1+1] = 0; } #else if ( dirc[l1-1] != '/' ) { dirc[l1] = '/'; dirc[l1+1] = 0; } #endif + */ ss = strrchr( name, '.' ); /* find last / */ ss++; strcpy(ext,ss); /* save extension */ @@ -342,31 +357,31 @@ void free_vector(double*v, int nl, int nh) } /************************ivector *******************************/ -char *cvector(long nl,long nh) +int *ivector(long nl,long nh) { - char *v; - v=(char *) malloc((size_t)((nh-nl+1+NR_END)*sizeof(char))); - if (!v) nrerror("allocation failure in cvector"); + int *v; + v=(int *) malloc((size_t)((nh-nl+1+NR_END)*sizeof(int))); + if (!v) nrerror("allocation failure in ivector"); return v-nl+NR_END; } /******************free ivector **************************/ -void free_cvector(char *v, long nl, long nh) +void free_ivector(int *v, long nl, long nh) { free((FREE_ARG)(v+nl-NR_END)); } -/************************ivector *******************************/ -int *ivector(long nl,long nh) +/************************lvector *******************************/ +long *lvector(long nl,long nh) { - int *v; - v=(int *) malloc((size_t)((nh-nl+1+NR_END)*sizeof(int))); + long *v; + v=(long *) malloc((size_t)((nh-nl+1+NR_END)*sizeof(long))); if (!v) nrerror("allocation failure in ivector"); return v-nl+NR_END; } -/******************free ivector **************************/ -void free_ivector(int *v, long nl, long nh) +/******************free lvector **************************/ +void free_lvector(long *v, long nl, long nh) { free((FREE_ARG)(v+nl-NR_END)); } @@ -425,7 +440,7 @@ double **matrix(long nrl, long nrh, long ncl, long nch) for (i=nrl+1; i<=nrh; i++) m[i]=m[i-1]+ncol; return m; - /* print *(*(m+1)+70) ou print m[1][70]; print m+1 or print &(m[1]) + /* print *(*(m+1)+70) or print m[1][70]; print m+1 or print &(m[1]) */ } @@ -1210,7 +1225,7 @@ double func( double *x) ipmx +=1; sw += weight[i]; ll[s[mw[mi][i]][i]] += 2*weight[i]*lli; - /*printf("i=%6d s1=%1d s2=%1d mi=%1d mw=%1d dh=%3d prob=%10.6f w=%6.4f out=%10.6f sav=%10.6f\n",i,s1,s2,mi,mw[mi][i],dh[mi][i],exp(lli),weight[i],out[s1][s2],savm[s1][s2]);*/ +/* printf("i=%6d s1=%1d s2=%1d mi=%1d mw=%1d dh=%3d prob=%10.6f w=%6.4f out=%10.6f sav=%10.6f\n",i,s1,s2,mi,mw[mi][i],dh[mi][i],exp(lli),weight[i],out[s1][s2],savm[s1][s2]); */ } /* end of wave */ } /* end of individual */ }else{ /* ml=5 no inter-extrapolation no jackson =0.8a */ @@ -1248,10 +1263,114 @@ double func( double *x) for(k=1,l=0.; k<=nlstate; k++) l += ll[k]; /* printf("l1=%f l2=%f ",ll[1],ll[2]); */ l= l*ipmx/sw; /* To get the same order of magnitude as if weight=1 for every body */ - /*exit(0); */ return -l; } +/*************** log-likelihood *************/ +double funcone( double *x) +{ + int i, ii, j, k, mi, d, kk; + double l, ll[NLSTATEMAX], cov[NCOVMAX]; + double **out; + double lli; /* Individual log likelihood */ + int s1, s2; + double bbh, survp; + /*extern weight */ + /* We are differentiating ll according to initial status */ + /* for (i=1;i<=npar;i++) printf("%f ", x[i]);*/ + /*for(i=1;i nlstate && (mle <5) ){ /* Jackson */ + lli=log(out[s1][s2] - savm[s1][s2]); + } else if (mle==1){ + lli= log((1.+bbh)*out[s1][s2]- bbh*savm[s1][s2]); /* linear interpolation */ + } else if(mle==2){ + lli= (savm[s1][s2]>(double)1.e-8 ?log((1.+bbh)*out[s1][s2]- bbh*savm[s1][s2]):log((1.+bbh)*out[s1][s2])); /* linear interpolation */ + } else if(mle==3){ /* exponential inter-extrapolation */ + lli= (savm[s1][s2]>(double)1.e-8 ?(1.+bbh)*log(out[s1][s2])- bbh*log(savm[s1][s2]):log((1.+bbh)*out[s1][s2])); /* exponential inter-extrapolation */ + } else if (mle==4){ /* mle=4 no inter-extrapolation */ + lli=log(out[s1][s2]); /* Original formula */ + } else{ /* ml>=5 no inter-extrapolation no jackson =0.8a */ + lli=log(out[s1][s2]); /* Original formula */ + } /* End of if */ + ipmx +=1; + sw += weight[i]; + ll[s[mw[mi][i]][i]] += 2*weight[i]*lli; +/* printf("i=%6d s1=%1d s2=%1d mi=%1d mw=%1d dh=%3d prob=%10.6f w=%6.4f out=%10.6f sav=%10.6f\n",i,s1,s2,mi,mw[mi][i],dh[mi][i],exp(lli),weight[i],out[s1][s2],savm[s1][s2]); */ + if(globpr){ + fprintf(ficresilk,"%6d %1d %1d %1d %1d %3d %10.6f %6.4f %10.6f %10.6f %10.6f ", \ + i,s1,s2,mi,mw[mi][i],dh[mi][i],exp(lli),weight[i],2*weight[i]*lli,out[s1][s2],savm[s1][s2]); + for(k=1,l=0.; k<=nlstate; k++) + fprintf(ficresilk," %10.6f",ll[k]); + fprintf(ficresilk,"\n"); + } + } /* end of wave */ + } /* end of individual */ + for(k=1,l=0.; k<=nlstate; k++) l += ll[k]; + /* printf("l1=%f l2=%f ",ll[1],ll[2]); */ + l= l*ipmx/sw; /* To get the same order of magnitude as if weight=1 for every body */ + return -l; +} + + +void likelione(FILE *ficres,double p[], int npar, int nlstate, int *globpr, long *ipmx, double *sw, double *fretone, double (*funcone)(double [])) +{ + /* This routine should help understanding what is done with the selection of individuals/waves and + to check the exact contribution to the likelihood. + Plotting could be done. + */ + int k; + if(globpr !=0){ /* Just counts and sums no printings */ + strcpy(fileresilk,"ilk"); + strcat(fileresilk,fileres); + if((ficresilk=fopen(fileresilk,"w"))==NULL) { + printf("Problem with resultfile: %s\n", fileresilk); + fprintf(ficlog,"Problem with resultfile: %s\n", fileresilk); + } + fprintf(ficresilk, "# individual(line's record) s1 s2 wave# effective_wave# number_of_product_matrix pij weight 2ln(pij)*weight 0pij_x 0pij_(x-stepm) cumulating_loglikeli_by_health_state"); + fprintf(ficresilk, "# i s1 s2 mi mw dh likeli weight out sav "); + /* i,s1,s2,mi,mw[mi][i],dh[mi][i],exp(lli),weight[i],2*weight[i]*lli,out[s1][s2],savm[s1][s2]); */ + for(k=1; k<=nlstate; k++) + fprintf(ficresilk," ll[%d]",k); + fprintf(ficresilk,"\n"); + } + + *fretone=(*funcone)(p); + if(globpr !=0) + fclose(ficresilk); + return; +} /*********** Maximum Likelihood Estimation ***************/ @@ -1260,6 +1379,7 @@ void mlikeli(FILE *ficres,double p[], int npar, int ncovmodel, int nlstate, doub int i,j, iter; double **xi; double fret; + double fretone; /* Only one call to likelihood */ char filerespow[FILENAMELENGTH]; xi=matrix(1,npar,1,npar); for (i=1;i<=npar;i++) @@ -1277,6 +1397,7 @@ void mlikeli(FILE *ficres,double p[], int npar, int ncovmodel, int nlstate, doub for(j=1;j<=nlstate+ndeath;j++) if(j!=i)fprintf(ficrespow," p%1d%1d",i,j); fprintf(ficrespow,"\n"); + powell(p,xi,npar,ftol,&iter,&fret,func); fclose(ficrespow); @@ -1829,11 +1950,11 @@ void concatwav(int wav[], int **dh, int **bh, int **mw, int **s, double *agedc wav[i]=mi; if(mi==0){ if(first==0){ - printf("Warning! None valid information for:%d line=%d (skipped) and may be others, see log file\n",num[i],i); + printf("Warning! None valid information for:%ld line=%d (skipped) and may be others, see log file\n",num[i],i); first=1; } if(first==1){ - fprintf(ficlog,"Warning! None valid information for:%d line=%d (skipped)\n",num[i],i); + fprintf(ficlog,"Warning! None valid information for:%ld line=%d (skipped)\n",num[i],i); } } /* end mi==0 */ } /* End individuals */ @@ -1845,15 +1966,21 @@ void concatwav(int wav[], int **dh, int **bh, int **mw, int **s, double *agedc else{ if (s[mw[mi+1][i]][i] > nlstate) { /* A death */ if (agedc[i] < 2*AGESUP) { - j= rint(agedc[i]*12-agev[mw[mi][i]][i]*12); - if(j==0) j=1; /* Survives at least one month after exam */ - k=k+1; - if (j >= jmax) jmax=j; - if (j <= jmin) jmin=j; - sum=sum+j; - /*if (j<0) printf("j=%d num=%d \n",j,i);*/ - /* printf("%d %d %d %d\n", s[mw[mi][i]][i] ,s[mw[mi+1][i]][i],j,i);*/ - if(j<0)printf("Error! Negative delay (%d to death) between waves %d and %d of individual %d at line %d who is aged %.1f with statuses from %d to %d\n ",j,mw[mi][i],mw[mi+1][i],num[i], i,agev[mw[mi][i]][i],s[mw[mi][i]][i] ,s[mw[mi+1][i]][i]); + j= rint(agedc[i]*12-agev[mw[mi][i]][i]*12); + if(j==0) j=1; /* Survives at least one month after exam */ + else if(j<0){ + printf("Error! Negative delay (%d to death) between waves %d and %d of individual %ld at line %d who is aged %.1f with statuses from %d to %d\n ",j,mw[mi][i],mw[mi+1][i],num[i], i,agev[mw[mi][i]][i],s[mw[mi][i]][i] ,s[mw[mi+1][i]][i]); + j=1; /* Careful Patch */ + printf(" We assumed that the date of interview was correct (and not the date of death) and postponed the death %d month(s) (one stepm) after the interview.\n You MUST fixe the contradiction between dates.\n",stepm); + printf("Error! Negative delay (%d to death) between waves %d and %d of individual %ld at line %d who is aged %.1f with statuses from %d to %d\n ",j,mw[mi][i],mw[mi+1][i],num[i], i,agev[mw[mi][i]][i],s[mw[mi][i]][i] ,s[mw[mi+1][i]][i]); + fprintf(ficlog," We assumed that the date of interview was correct (and not the date of death) and postponed the death %d month(s) (one stepm) after the interview.\n You MUST fix the contradiction between dates.\n",stepm); + } + k=k+1; + if (j >= jmax) jmax=j; + if (j <= jmin) jmin=j; + sum=sum+j; + /*if (j<0) printf("j=%d num=%d \n",j,i);*/ + /* printf("%d %d %d %d\n", s[mw[mi][i]][i] ,s[mw[mi+1][i]][i],j,i);*/ } } else{ @@ -1864,13 +1991,16 @@ void concatwav(int wav[], int **dh, int **bh, int **mw, int **s, double *agedc else if (j <= jmin)jmin=j; /* if (j<10) printf("j=%d jmin=%d num=%d ",j,jmin,i); */ /*printf("%d %lf %d %d %d\n", i,agev[mw[mi][i]][i],j,s[mw[mi][i]][i] ,s[mw[mi+1][i]][i]);*/ - if(j<0)printf("Error! Negative delay (%d) between waves %d and %d of individual %d at line %d who is aged %.1f with statuses from %d to %d\n ",j,mw[mi][i],mw[mi+1][i],num[i], i,agev[mw[mi][i]][i],s[mw[mi][i]][i] ,s[mw[mi+1][i]][i]); + if(j<0){ + printf("Error! Negative delay (%d) between waves %d and %d of individual %ld at line %d who is aged %.1f with statuses from %d to %d\n ",j,mw[mi][i],mw[mi+1][i],num[i], i,agev[mw[mi][i]][i],s[mw[mi][i]][i] ,s[mw[mi+1][i]][i]); + fprintf(ficlog,"Error! Negative delay (%d) between waves %d and %d of individual %ld at line %d who is aged %.1f with statuses from %d to %d\n ",j,mw[mi][i],mw[mi+1][i],num[i], i,agev[mw[mi][i]][i],s[mw[mi][i]][i] ,s[mw[mi+1][i]][i]); + } sum=sum+j; } jk= j/stepm; jl= j -jk*stepm; ju= j -(jk+1)*stepm; - if(mle <=1){ + if(mle <=1){ /* only if we use a the linear-interpoloation pseudo-likelihood */ if(jl==0){ dh[mi][i]=jk; bh[mi][i]=0; @@ -1895,8 +2025,8 @@ void concatwav(int wav[], int **dh, int **bh, int **mw, int **s, double *agedc bh[mi][i]=ju; /* At least one step */ /* printf(" bh=%d ju=%d jl=%d dh=%d jk=%d stepm=%d %d\n",bh[mi][i],ju,jl,dh[mi][i],jk,stepm,i);*/ } - } - } /* end if mle */ + } /* end if mle */ + } } /* end wave */ } jmean=sum/k; @@ -2856,11 +2986,11 @@ void printinghtml(char fileres[], char title[], char datafile[], int firstpass, fprintf(ficlog,"Problem with %s \n",optionfilehtm), exit(0); } - fprintf(fichtm,"