Next: Module: Genetic Backpropagation
Up: Source Code
Previous: Module: Genetic Algorithm
Contents
Subsections
1
2 /* Declarations of backpropagation algorithm */
3
4 #ifndef BACK_H
5 #define BACK_H 1
6
7 #include "defs.h"
8
9 char *BackOptStr();
10 char *BackUsage();
11 char BackParamStr[256];
12
13 float NetErr; /* residual Error */
14 float LearnRate; /* learn rate */
15 float Impulse; /* impulse constant */
16 float InitWeight; /* initial weight range */
17
18 /* Arrays */
19
20 float **Wih; /* weight matrix input -> hidden */
21 float **Who; /* weight matrix hidden -> output */
22 float *Hid; /* hidden states */
23 float *Out; /* output states */
24 float *BiasHid; /* bias of hidden neurons */
25 float *BiasOut; /* bias of output neurons */
26 float **Dih,**Dho; /* Weight update (1st order) */
27 float **DDih,**DDho; /* Weight update (2nd order) */
28
29 /* Procedures */
30
31 int handleBackOpt(char opt,char* arg);
32 int initBack();
33 void randomNet(float w);
34 float calcNetErr();
35 void updateNet();
36 void printnet();
37
38 #endif
1
2 /* Implementation of backpropagation algorithm */
3
4 #include <math.h>
5
6 #include "defs.h"
7 #include "back.h"
8 #include "ind.h"
9
10 /* Parameter Handling */
11
12 #define DEFLEARNRATE 1.0
13 #define DEFIMPULSE 0.0
14 #define DEFINITWEIGHT 1.0
15 #define DEFONLINE 1
16
17 char *BackOptStr() {return "o:w:k:a:\0";}
18
19 char *BackUsage() {return
20 "Backpropagation Parameters:\n"
21 "-o <use online learning ?>: 1\n"
22 "-w <w for initial weights [-w,w]> 1.0\n"
23 "-k <learn rate>: 1.0\n"
24 "-a <impulse factor>: 0.0\n\0";}
25
26 /* set default values */
27
28 int Online =DEFONLINE; /* Use online learning */
29 float LearnRate =DEFLEARNRATE; /* learn rate */
30 float Impulse =DEFIMPULSE; /* impulse constant */
31 float InitWeight=DEFINITWEIGHT; /* initial weight range */
32
33 /* others */
34
35 float *Dout; /* Delta update for output layer */
36
37 const char OnlineStr[2][10]={"Batch","Online"};
38
39 int handleBackOpt(char opt,char* arg)
40 {
41 switch(opt)
42 {
43 case 'o': return (Online =getint(arg,0,1))<0;
44 case 'w': return getfloat(&InitWeight,arg,0.0,1000.0);
45 case 'k': return getfloat(&LearnRate,arg,0.0,1000.0);
46 case 'a': return getfloat(&Impulse,arg,0.0,1000.0);
47 default: return 1;
48 };
49 }
50
51 int initBack()
52 {
53 int i,j,k;
54 float *p,*q,*pp,*qq;
55
56 BackCalcs=0;
57 if(!(Hid=(float*)malloc((Nin+1)*FLOATLEN))) return 1;
58 if(!(Out=(float*)malloc(Nout*FLOATLEN))) return 1;
59 if(!(Dout=(float*)malloc(Nout*FLOATLEN))) return 1;
60 if(!(Wih=(float**)malloc((Nin+1)*PNTLEN))) return 1;
61 if(!(Who=(float**)malloc((Nhid+1)*PNTLEN))) return 1;
62 if(!(Dih=(float**)malloc((Nin+1)*PNTLEN))) return 1;
63 if(!(Dho=(float**)malloc((Nhid+1)*PNTLEN))) return 1;
64 if(!(p=(float*)malloc((Nin+1)*Nhid*FLOATLEN))) return 1;
65 if(!(q=(float*)malloc((Nhid+1)*Nout*FLOATLEN))) return 1;
66 if(!(pp=(float*)malloc((Nin+1)*Nhid*FLOATLEN))) return 1;
67 if(!(qq=(float*)malloc((Nhid+1)*Nout*FLOATLEN))) return 1;
68 for(i=0;i<=Nin;i++)
69 {
70 Wih[i]=p; p+=Nhid;
71 Dih[i]=pp; pp+=Nhid;
72 for(j=0;j<Nhid;j++) Dih[i][j]=0.0;
73 };
74 BiasHid=Wih[Nin];
75 for(j=0;j<=Nhid;j++)
76 {
77 Who[j]=q; q+=Nout;
78 Dho[j]=qq; qq+=Nout;
79 for(k=0;k<Nout;k++) Dho[j][k]=0.0;
80 };
81 randomNet(InitWeight);
82 BiasOut=Who[Nhid];
83 Hid[Nhid]=1.0;
84
85 if(Nback || Impulse!=0.0)
86 {
87 if(!(DDih=(float**)malloc((Nin+1)*PNTLEN))) return 1;
88 if(!(DDho=(float**)malloc((Nhid+1)*PNTLEN))) return 1;
89 if(!(p=(float*)malloc((Nin+1)*Nhid*FLOATLEN))) return 1;
90 if(!(q=(float*)malloc((Nhid+1)*Nout*FLOATLEN))) return 1;
91 for(i=0;i<=Nin;i++) { DDih[i]=p; p+=Nhid; };
92 for(j=0;j<=Nhid;j++) { DDho[j]=q; q+=Nout; };
93 };
94 if(Nback)
95 sprintf(BackParamStr,
96 "Backprop.: Method: %s, InitWeight = [%7.3f,%7.3f]\n",
97 OnlineStr[Online],-InitWeight,+InitWeight);
98 else
99 sprintf(BackParamStr,
100 "Backprop.: Method: %s, LearnRate = %7.4f, Impulse = %7.4f\n"
101 " InitWeight = [%7.3f,%7.3f]\n",
102 OnlineStr[Online],LearnRate,Impulse,
103 -InitWeight,+InitWeight);
104 return 0;
105 }
106
107 #define randweight() (((float)getrand()/(float)MAXRANDOM) \
108 *2.0*w-w)
109
110 void randomNet(float w)
111 {
112 int i,j,k;
113
114 for(i=0;i<=Nin;i++)
115 for(j=0;j<Nhid;j++) Wih[i][j]= randweight();
116 for(j=0;j<=Nhid;j++)
117 for(k=0;k<Nout;k++) Who[j][k]= randweight();
118 }
119
120 #undef randweight
121
122 #define sigma(x) 1/(1+exp(-(x)))
123
124 float evaluate(float *in,float *out)
125 {
126 int i,j,k;
127 float s,e,d;
128
129 for(j=0;j<Nhid;j++)
130 {
131 s=0.0;
132 for(i=0;i<=Nin;i++) s+=in[i]*Wih[i][j];
133 Hid[j]=sigma(s);
134 };
135 for(k=0;k<Nout;k++)
136 {
137 s=0.0;
138 for(j=0;j<=Nhid;j++) s+=Hid[j]*Who[j][k];
139 Out[k]=sigma(s);
140 };
141 e=0.0;
142 for(k=0;k<Nout;k++)
143 {
144 d=(out[k]-Out[k]);
145 Dout[k]=d*Out[k]*(1-Out[k]);
146 e+=d*d;
147 };
148 return 0.5*e;
149 }
150
151 #undef sigma
152
153 #define swap(x,y) {pt=x; x=y; y=pt;}
154
155 float calcNetErr()
156 {
157 int i,j,k,p;
158 float **pt;
159 float *in,*out;
160 float s;
161 NetErr=0.0;
162
163 BackCalcs+=Ntrain;
164 if(Online)
165 {
166 for(p=0;p<Ntrain;p++)
167 {
168 if(Impulse!=0) { swap(Dih,DDih); swap(Dho,DDho); };
169 in=TrainIn[p]; out=TrainOut[p];
170 NetErr+=evaluate(in,out);
171 for(j=0;j<Nhid;j++)
172 {
173 s=0.0;
174 for(k=0;k<Nout;k++) s+=Who[j][k]*Dout[k];
175 s*=Hid[j]*(1-Hid[j]);
176 if(Impulse!=0.0)
177 for(i=0;i<=Nin;i++)
178 Wih[i][j]+=Dih[i][j]=
179 LearnRate*s*in[i]+Impulse*DDih[i][j];
180 else
181 for(i=0;i<=Nin;i++)
182 Wih[i][j]+=LearnRate*s*in[i];
183 };
184 for(j=0;j<=Nhid;j++)
185 {
186 if(Impulse!=0.0)
187 {
188 for(k=0;k<Nout;k++)
189 Who[j][k]+=Dho[j][k]=
190 LearnRate*Dout[k]*Hid[j]+Impulse*DDho[j][k];
191 } else {
192 for(k=0;k<Nout;k++)
193 Who[j][k]+=LearnRate*Dout[k]*Hid[j];
194 };
195 };
196 };
197 NetErr/=Ntrain;
198 return NetErr;
199 } else {
200 if(Impulse!=0) { swap(Dih,DDih); swap(Dho,DDho); };
201
202 for(i=0;i<=Nin;i++)
203 for(j=0;j<Nhid;j++) Dih[i][j]=0.0;
204 for(j=0;j<=Nhid;j++)
205 for(k=0;k<Nout;k++) Dho[j][k]=0.0;
206
207 for(p=0;p<Ntrain;p++)
208 {
209 in=TrainIn[p]; out=TrainOut[p];
210 NetErr+=evaluate(in,out);
211 for(j=0;j<Nhid;j++)
212 {
213 s=0.0;
214 for(k=0;k<Nout;k++) s+=Who[j][k]*Dout[k];
215 s*=Hid[j]*(1-Hid[j]);
216 for(i=0;i<=Nin;i++)
217 Dih[i][j]+=s*in[i];
218 };
219 for(j=0;j<=Nhid;j++)
220 for(k=0;k<Nout;k++)
221 Dho[j][k]+=Dout[k]*Hid[j];
222 };
223 NetErr/=Ntrain;
224 return NetErr;
225 };
226 }
227
228 #undef swap
229
230 void updateNet()
231 {
232 int i,j,k;
233 float l;
234
235 if(!Online)
236 {
237 l=LearnRate/Ntrain;
238 if(Impulse!=0.0)
239 {
240 for(i=0;i<=Nin;i++)
241 for(j=0;j<Nhid;j++)
242 {
243 Dih[i][j]*=l;
244 Wih[i][j]+=Dih[i][j]+=Impulse*DDih[i][j];
245 };
246 for(j=0;j<=Nhid;j++)
247 for(k=0;k<Nout;k++)
248 {
249 Dho[j][k]*=l;
250 Who[j][k]+=Dho[j][k]+=Impulse*DDho[j][k];
251 };
252 } else {
253 for(i=0;i<=Nin;i++)
254 for(j=0;j<Nhid;j++) Wih[i][j]+=Dih[i][j]*=l;
255 for(j=0;j<=Nhid;j++)
256 for(k=0;k<Nout;k++) Who[j][k]+=Dho[j][k]*=l;
257 };
258 };
259 }
260
261 void printnet()
262 {
263 int i,j,k,c;
264
265 c=0;
266 for(j=0;j<Nhid;j++)
267 {
268 if(c>70-6*Nin) { printf("\n"); c=0; };
269 printf("(");
270 for(i=0;i<Nin;i++) printf("%5.2f,",Wih[i][j]);
271 printf("b:%5.2f)",Wih[Nin][j]);
272 c+=9+6*Nin;
273 };
274 printf("\n"); c=0;
275 for(k=0;k<Nout;k++)
276 {
277 if(c>70-6*Nhid) { printf("\n"); c=0; };
278 printf("(");
279 for(j=0;j<Nhid;j++) printf("%5.2f,",Who[j][k]);
280 printf("b:%5.2f)",Who[Nhid][k]);
281 c+=9+6*Nhid;
282 };
283 printf("\n");
284 }
(c) Bernhard Ömer - oemer@tph.tuwien.ac.at - http://tph.tuwien.ac.at/~oemer/