metropolis

 stephane.adjemian@ens.fr [09-02-2005]

0001 function metropolis(xparam1,vv,gend,data,rawdata,mh_bounds)
0002 % stephane.adjemian@ens.fr [09-02-2005]
0003 global M_ oo_ options_ bayestopt_ estim_params_
0004 
0005 bayestopt_.penalty = 1e8;
0006 
0007 MhDirectoryName = CheckPath('metropolis');
0008 
0009 nblck = options_.mh_nblck;
0010 nruns = ones(nblck,1)*options_.mh_replic;
0011 npar  = length(xparam1);
0012 MAX_nruns = ceil(options_.MaxNumberOfBytes/(npar+2)/8);
0013 d = chol(vv);
0014 options_.lik_algo = 1;
0015 OpenOldFile = ones(nblck,1);
0016 
0017 if options_.load_mh_file == 0
0018   % Here we start a new metropolis-hastings, previous draws are not
0019   % considered.
0020   if nblck > 1
0021     disp('MH: Multiple chains mode.')
0022   else
0023     disp('MH: One Chain mode.')
0024   end
0025   % Delete old mh files...
0026   files = dir([ MhDirectoryName '/' M_.fname '_mh*_blck*.mat']);
0027   if length(files)
0028     delete([ MhDirectoryName '/' M_.fname '_mh*_blck*.mat']);
0029     disp('MH: Old _mh files succesfully erased!')
0030   end
0031   % Initial values...
0032   if nblck > 1% Case 1: multiple chains
0033     disp('MH: Searching for initial values...')
0034     ix2 = zeros(nblck,npar);
0035     ilogpo2 = zeros(nblck,1);
0036     for j=1:nblck
0037       validate    = 0;
0038       init_iter    = 0;
0039       trial = 1;
0040       while validate == 0 & trial <= 10 
0041     candidate = options_.mh_init_scale*randn(1,npar)*d + transpose(xparam1);
0042     if all(candidate' > mh_bounds(:,1)) & all(candidate' < mh_bounds(:,2)) 
0043       ix2(j,:) = candidate;
0044       ilogpo2(j) = -DsgeLikelihood(ix2(j,:)',gend,data);
0045       j = j+1;
0046       validate = 1;
0047     end
0048     init_iter = init_iter + 1;
0049     if init_iter > 100 & validate == 0
0050       disp(['MH: I couldn''t get a valid initial value in 100 trials.'])
0051       disp(['MH: You should Reduce mh_init_scale...'])
0052       disp(sprintf('MH: Parameter mh_init_scale is equal to %f.',options_.mh_init_scale))
0053       options_.mh_init_scale = input('MH: Enter a new value...  ');
0054       trial = trial+1;
0055     end
0056       end
0057       if trial > 10 & ~validate
0058     disp(['MH: I''m unable to find a starting value for block ' int2str(j)])
0059     return
0060       end
0061     end
0062     disp('MH: Initial values found!')
0063     disp(' ')
0064   else% Case 2: one chain (we start from the posterior mode)
0065     candidate = transpose(xparam1);
0066     if all(candidate' > mh_bounds(:,1)) & all(candidate' < mh_bounds(:,2)) 
0067       ix2 = candidate;
0068       ilogpo2 = -DsgeLikelihood(ix2',gend,data);
0069       disp('MH: Initialization at the posterior mode.')
0070       disp(' ')
0071     else
0072       disp('MH: Initialization failed...')
0073       disp('MH: The posterior mode lies outside the prior bounds.')
0074       return
0075     end
0076   end
0077   fblck = 1;
0078   fline = ones(nblck,1);
0079   NewFile = ones(nblck,1);
0080   % Creation of the mh-history file:
0081   file = dir([MhDirectoryName '/'  M_.fname '_mh_history.mat']);
0082   if length(files)
0083     delete([ MhDirectoryName '/' M_.fname '_mh_history.mat']);
0084     disp('MH: Old mh_history file succesfully erased!')
0085   end
0086   AnticipatedNumberOfFiles = floor(nruns(1)/MAX_nruns);
0087   AnticipatedNumberOfLinesInTheLastFile = nruns(1) - AnticipatedNumberOfFiles*MAX_nruns;
0088   record.Nblck = nblck;
0089   record.MhDraws = zeros(1,3);
0090   record.MhDraws(1,1) = nruns(1);
0091   record.MhDraws(1,2) = AnticipatedNumberOfFiles+1;
0092   record.MhDraws(1,3) = AnticipatedNumberOfLinesInTheLastFile;
0093   record.AcceptationRates = zeros(1,nblck);
0094   record.Seeds.Normal = randn('state');
0095   record.Seeds.Unifor = rand('state');
0096   record.InitialParameters = ix2;
0097   record.InitialLogLiK = ilogpo2;
0098   record.LastParameters = zeros(nblck,npar);
0099   record.LastLogLiK = zeros(nblck,1);
0100   record.LastFileNumber = AnticipatedNumberOfFiles+1;
0101   record.LastLineNumber = AnticipatedNumberOfLinesInTheLastFile;
0102   save([MhDirectoryName '/' M_.fname '_mh_history'],'record');  
0103 elseif options_.load_mh_file == 1% Here we consider previous mh files (previous mh did not crash).
0104   disp('MH: I''m loading past metropolis-hastings simulations...')
0105   file = dir([ MhDirectoryName '/'  M_.fname '_mh_history.mat' ]);
0106   files = dir([ MhDirectoryName '/' M_.fname '_mh*.mat']);
0107   if ~length(files)
0108     disp('MH:: FAILURE! there is no MH file to load here!')
0109     return  
0110   end
0111   if ~length(file)
0112     disp('MH:: FAILURE! there is no MH-history file!')
0113     return
0114   else
0115     load([ MhDirectoryName '/'  M_.fname '_mh_history'])
0116   end
0117   past_number_of_blocks = record.Nblck;
0118   if past_number_of_blocks ~= nblck
0119     disp('MH:: The specified number of blocks doesn''t match with the previous number of blocks!')
0120     disp(['MH:: You declared ' int2str(nblck) ' blocks, but the previous number of blocks was ' int2str(past_number_of_blocks) '.'])
0121     disp(['MH:: I will run the Metropolis-Hastings with ' int2str(past_number_of_blocks) ' blocks.' ])
0122     nblck = past_number_of_blocks;
0123     options_.mh_nblck = nblck;
0124   end
0125   % I get the last line of the last mh-file for initialization
0126   % of the new metropolis-hastings simulations:
0127   LastFileNumber = record.LastFileNumber;
0128   LastLineNumber = record.LastLineNumber;
0129   if LastLineNumber < MAX_nruns
0130     NewFile = ones(nblck,1)*LastFileNumber;
0131   else
0132     NewFile = ones(nblck,1)*(LastFileNumber+1);
0133   end  
0134   ilogpo2 = record.LastLogLiK;
0135   ix2 = record.LastParameters;
0136   fblck = 1;
0137   fline = ones(nblck,1)*(LastLineNumber+1);
0138   NumberOfPreviousSimulations = sum(record.MhDraws(:,1),1);
0139   record.MhDraws = [record.MhDraws;zeros(1,3)];
0140   NumberOfDrawsWrittenInThePastLastFile = MAX_nruns - LastLineNumber;
0141   NumberOfDrawsToBeSaved = nruns(1) - NumberOfDrawsWrittenInThePastLastFile;
0142   AnticipatedNumberOfFiles = floor(NumberOfDrawsToBeSaved/MAX_nruns);
0143   AnticipatedNumberOfLinesInTheLastFile = NumberOfDrawsToBeSaved - AnticipatedNumberOfFiles*MAX_nruns;  
0144   record.LastFileNumber = LastFileNumber + AnticipatedNumberOfFiles + 1;
0145   record.LastLineNumber = AnticipatedNumberOfLinesInTheLastFile;
0146   record.MhDraws(end,1) = nruns(1);
0147   record.MhDraws(end,2) = AnticipatedNumberOfFiles+1;
0148   record.MhDraws(end,3) = AnticipatedNumberOfLinesInTheLastFile;
0149   randn('state',record.Seeds.Normal);
0150   rand('state',record.Seeds.Unifor);
0151   save([MhDirectoryName '/' M_.fname '_mh_history'],'record');
0152   disp(['MH: ... It''s done. I''ve loaded ' int2str(NumberOfPreviousSimulations) ' simulations.'])
0153   disp(' ')
0154 elseif options_.load_mh_file == -1% The previous metropolis-hastings
0155                                   % crashed before the end! I try to
0156                                   % recover the saved draws...
0157   disp('MH: Recover mode!')
0158   disp(' ')
0159   file = dir([MhDirectoryName '/'  M_.fname '_mh_history.mat']);
0160   if ~length(file)
0161     disp('MH:: FAILURE! there is no MH-history file!')
0162     return
0163   else
0164     load([ MhDirectoryName '/'  M_.fname '_mh_history'])
0165   end
0166   nblck = record.Nblck;
0167   options_.mh_nblck = nblck;
0168   if size(record.MhDraws,1) == 1
0169     OldMh = 0;% The crashed metropolis was the first session.
0170   else
0171     OldMh = 1;% The crashed metropolis wasn't the first session.
0172   end
0173   % Default initialization:
0174   if OldMh
0175     ilogpo2 = record.LastLogLiK;
0176     ix2 = record.LastParameters;
0177   else
0178     ilogpo2 = record.InitialLogLiK;
0179     ix2 = record.InitialParameters;
0180   end
0181   % Set "NewFile":
0182   if OldMh
0183     LastLineNumberInThePreviousMh = record.MhDraws(end-1,3);
0184     LastFileNumberInThePreviousMh = sum(record.MhDraws(1:end-1,2),1);
0185     if LastLineNumberInThePreviousMh < MAX_nruns
0186       NewFile = ones(nblck,1)*LastFileNumberInThePreviousMh;
0187     else
0188       NewFile = ones(nblck,1)*(LastFileNumberInThePreviousMh+1);
0189     end
0190   else
0191     NewFile = ones(nblck,1);
0192   end
0193   % Set fline (First line):
0194   if OldMh
0195     fline = ones(nblck,1)*(record.MhDraws(end-1,3)+1);
0196   else
0197     fline = ones(nblck,1);
0198   end
0199   % Set fblck (First block):
0200   fblck = 1;
0201   % How many mh files should we have ?
0202   ExpectedNumberOfMhFilesPerBlock = sum(record.MhDraws(:,2),1);
0203   ExpectedNumberOfMhFiles = ExpectedNumberOfMhFilesPerBlock*nblck;
0204   % I count the total number of saved mh files...
0205   AllMhFiles = dir([MhDirectoryName '/' M_.fname '_mh*_blck*.mat']);
0206   TotalNumberOfMhFiles = length(AllMhFiles);
0207   % I count the number of saved mh files per block
0208   NumberOfMhFilesPerBlock = zeros(nblck,1);
0209   for i = 1:nblck
0210     BlckMhFiles = dir([ MhDirectoryName '/' M_.fname '_mh*_blck' int2str(i) '.mat']);
0211     NumberOfMhFilesPerBlock(i) = length(BlckMhFiles);
0212   end
0213   tmp = NumberOfMhFilesPerBlock(1); b = 1;
0214   % Is there a chain with less mh files than the first chain ?
0215   CrashedBlck = 1; 
0216   while b <= nblck
0217     if  NumberOfMhFilesPerBlock(b) < ExpectedNumberOfMhFilesPerBlock
0218       CrashedBlck = b;% YES!
0219       break
0220     end
0221     b = b+1;
0222   end
0223   % The new metropolis-hastings should start from (fblck=CrashedBlck)
0224   fblck = CrashedBlck;
0225   % How many mh-files are saved in this block ?
0226   NumberOfSavedMhFilesInTheCrashedBlck = ...
0227       NumberOfMhFilesPerBlock(CrashedBlck);
0228   % How many mh-files were saved in this block during the last session
0229   % (if there was a complete session before the crash) ?
0230   if OldMh
0231     ante = sum(record.MhDraws(1:end-1,2),1);
0232     load([MhDirectoryName '/' M_.fname '_mh' int2str(ante) '_blck' int2str(CrashedBlck) '.mat'],'logpo2');
0233     if length(logpo2) == MAX_nruns
0234       IsTheLastFileOfThePreviousMhFull = 1;
0235     else
0236       IsTheLastFileOfThePreviousMhFull = 0;
0237     end
0238   else
0239     ante = 0;% Because the crashed session is the first one
0240     IsTheLastFileOfThePreviousMhFull = 1;
0241   end
0242   if ~IsTheLastFileOfThePreviousMhFull
0243     MhFileExist  = 1;
0244     MhFileNumber = ante;
0245     while MhFileExist
0246       MhFileNumber = MhFileNumber + 1;
0247       if ~exist([MhDirectoryName '/' M_.fname '_mh' int2str(MhFileNumber) '_blck' int2str(CrashedBlck) '.mat'])
0248     MhFileExist = 0;
0249       end
0250     end
0251     % if MhFileNumber > ExpectedNumberOfMhFilesPerBlock % Peut tre  dplacer plus haut...
0252     %  disp('MH : You are using the recover mode but the previous session');
0253     %  disp('     of the metropolis-hastings didn''t crash!')
0254     %  disp('MH : I stop and you shoud modify your mod file.')
0255     %  return
0256     % else
0257     NumberOfCompletedMhFiles = (MhFileNumber-1)-ante;
0258     % How many runs were saved ?
0259     if OldMh
0260       reste = MAX_nruns-record.MhDraws(end-1,3);
0261     else
0262       reste = 0
0263     end
0264     NumberOfSavedDraws = MAX_nruns*(NumberOfCompletedMhFiles) + reste;
0265     % Here is the number of draws we still need to complete the block:
0266     nruns(CrashedBlck) = nruns(CrashedBlck)-NumberOfSavedDraws; 
0267     % I initialize with the last saved mh file of the inccomplete
0268     % block:
0269     load([MhDirectoryName '/' M_.fname '_mh' int2str(MhFileNumber-1) '_blck' int2str(CrashedBlck) '.mat']);
0270     ilogpo2(CrashedBlck) = logpo2(end);
0271     ix2(CrashedBlck,:) = x2(end,:);
0272     NewFile(CrashedBlck) = MhFileNumber;
0273     fline(CrashedBlck,1) = 1;
0274   else
0275     % creuser ce qui se passe dans ce cas !
0276     OpenOldFile(CrashedBlck) = 0;
0277     disp('Ok')
0278   end
0279 end% of (if options_.load_mh_file == {0,1 or -1})
0280 %%%%
0281 %%%% NOW i run the (nblck-fblck+1) metropolis-hastings chains
0282 %%%%
0283 InitSizeArray = min([MAX_nruns*ones(nblck) nruns],[],2);
0284 for b = fblck:nblck
0285   if (options_.load_mh_file~=0)  & (fline(b)>1) & OpenOldFile(b)
0286     load(['./' MhDirectoryName '/' M_.fname '_mh' int2str(NewFile(b)) ...
0287       '_blck' int2str(b) '.mat'])
0288     x2 = [x2;zeros(InitSizeArray(b)-fline(b)+1,npar)];
0289     logpo2 = [logpo2;zeros(InitSizeArray(b)-fline(b)+1,1)];
0290     OpenOldFile(b) = 0;
0291   else
0292     x2 = zeros(InitSizeArray(b),npar);
0293     logpo2 = zeros(InitSizeArray(b),1);
0294   end
0295   hh = waitbar(0,['Please wait... Metropolis-Hastings (' int2str(b) '/' int2str(nblck) ')...']);
0296   set(hh,'Name','Metropolis-Hastings')
0297   isux = 0;
0298   irun = fline(b);  
0299   j = 1;
0300   while j <= nruns(b)
0301     par = randn(1,npar)*d;
0302     par = par.*bayestopt_.jscale' + ix2(b,:);  
0303     if all(par'>mh_bounds(:,1)) & all(par'<mh_bounds(:,2))
0304       logpost = -DsgeLikelihood(par',gend,data);
0305     else
0306       logpost = -inf;
0307     end    
0308     if (logpost > -inf) & (log(rand) < logpost-ilogpo2(b))
0309       x2(irun,:) = par; 
0310       ix2(b,:) = par;
0311       logpo2(irun) = logpost; 
0312       ilogpo2(b) = logpost;
0313       isux = isux + 1;
0314     else    
0315       x2(irun,:) = ix2(b,:);
0316       logpo2(irun) = ilogpo2(b);
0317     end    
0318     prtfrc = j/nruns(b);
0319     waitbar(prtfrc,hh,[ '(' int2str(b) '/' int2str(nblck) ') ' sprintf('%f done, acceptation rate %f',prtfrc,isux/j)]);
0320     if (irun == InitSizeArray(b)) | (j == nruns(b)) % Now I save the simulations
0321       save([MhDirectoryName '/' M_.fname '_mh' int2str(NewFile(b)) '_blck' int2str(b)],'x2','logpo2');
0322       InitSizeArray(b) = min(nruns(b)-j,MAX_nruns);
0323       if j == nruns(b) % I record the last draw...
0324     record.LastParameters(b,:) = x2(end,:);
0325     record.LastLogLiK(b) = logpo2(end);
0326       end      
0327       if InitSizeArray(b)
0328     x2 = zeros(InitSizeArray(b),npar);
0329     logpo2 = zeros(InitSizeArray(b),1);
0330     NewFile(b) = NewFile(b) + 1;
0331     irun = 0;
0332       else % InitSizeArray is equal to zero because we are at the end of an mc chain.
0333     InitSizeArray(b) = min(nruns(b),MAX_nruns);
0334       end
0335     end
0336     j=j+1;
0337     irun = irun + 1;
0338   end% End of the simulations for one mh-block.
0339   record.AcceptationRates(b) = isux/j;
0340   close(hh);
0341 end% End of the loop over the mh-blocks.
0342 record.Seeds.Normal = randn('state');
0343 record.Seeds.Unifor = rand('state');
0344 save([MhDirectoryName '/' M_.fname '_mh_history'],'record');
0345 disp(['MH: Number of mh files                : ' int2str(NewFile(1)) ' per block.'])
0346 disp(['MH: Total number of generated files    : ' int2str(NewFile(1)*nblck) '.'])
0347 disp(['MH: Total number of iterations         : ' int2str((NewFile(1)-1)*MAX_nruns+irun-1) '.'])
0348 disp(' ')