function [rmse_MC, ixx] = filt_mc_(OutDir,options_gsa_,dataset_,dataset_info,M_,oo_,options_,bayestopt_,estim_params_) % [rmse_MC, ixx] = filt_mc_(OutDir,options_gsa_,dataset_,dataset_info,M_,oo_,options_,bayestopt_,estim_params_ % Inputs: % - OutputDirectoryName [string] name of the output directory % - options_gsa_ [structure] GSA options % - dataset_ [dseries] object storing the dataset % - dataset_info [structure] storing informations about the sample. % - M_ [structure] Matlab's structure describing the model % - oo_ [structure] storing the results % - options_ [structure] Matlab's structure describing the current options % - bayestopt_ [structure] describing the priors % - estim_params_ [structure] characterizing parameters to be estimated % % Outputs: % - rmse_MC [double] RMSE by nvar matrix of the RMSEs % - ixx [double] RMSE by nvar matrix of sorting % indices (descending order of RMSEs) % inputs (from opt_gsa structure) % vvarvecm = options_gsa_.var_rmse; % loadSA = options_gsa_.load_rmse; % pfilt = options_gsa_.pfilt_rmse; % alpha = options_gsa_.alpha_rmse; % alpha2 = options_gsa_.alpha2_rmse; % istart = options_gsa_.istart_rmse; % % Written by Marco Ratto % Joint Research Centre, The European Commission, % marco.ratto@ec.europa.eu % Copyright © 2012-2016 European Commission % Copyright © 2012-2023 Dynare Team % % This file is part of Dynare. % % Dynare is free software: you can redistribute it and/or modify % it under the terms of the GNU General Public License as published by % the Free Software Foundation, either version 3 of the License, or % (at your option) any later version. % % Dynare is distributed in the hope that it will be useful, % but WITHOUT ANY WARRANTY; without even the implied warranty of % MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the % GNU General Public License for more details. % % You should have received a copy of the GNU General Public License % along with Dynare. If not, see . vvarvecm = options_gsa_.var_rmse; if options_.TeX vvarvecm_tex = options_gsa_.var_rmse_tex; else vvarvecm_tex = {}; end loadSA = options_gsa_.load_rmse; pfilt = options_gsa_.pfilt_rmse; alpha = options_gsa_.alpha_rmse; alpha2 = 0; pvalue = options_gsa_.alpha2_rmse; istart = max(2,options_gsa_.istart_rmse); fname_ = M_.fname; skipline(1) disp('Starting sensitivity analysis') disp('for the fit of EACH observed series ...') skipline() if ~options_.nograph disp('Deleting old SA figures...') a=dir([OutDir,filesep,'*.*']); tmp1='0'; if options_.opt_gsa.ppost tmp='_rmse_post'; else if options_.opt_gsa.pprior tmp='_rmse_prior'; else tmp='_rmse_mc'; end if options_gsa_.lik_only tmp1 = [tmp,'_post_SA']; tmp = [tmp,'_lik_SA']; end end for j=1:length(a) if strmatch([fname_,tmp],a(j).name) if options_.debug disp(a(j).name) end delete([OutDir,filesep,a(j).name]) end if strmatch([fname_,tmp1],a(j).name) if options_.debug disp(a(j).name) end delete([OutDir,filesep,a(j).name]) end end disp('done !') end [param_names,param_names_tex]=get_LaTeX_parameter_names(M_,options_,estim_params_,bayestopt_); nshock=estim_params_.nvx + estim_params_.nvn + estim_params_.ncx + estim_params_.ncn; npar=estim_params_.np; if ~isempty(options_.mode_file) load(options_.mode_file,'xparam1') end if options_.opt_gsa.ppost c=load([M_.dname filesep 'Output' filesep fname_,'_mean.mat'],'xparam1'); xparam1_mean=c.xparam1; xparam1=c.xparam1; clear c elseif ~isempty(options_.mode_file) && exist([M_.dname filesep 'Output' filesep fname_,'_mean.mat'],'file')==2 c=load([M_.dname filesep 'Output' filesep fname_,'_mean.mat'],'xparam1'); xparam1_mean=c.xparam1; xparam1=c.xparam1; clear c end if options_.opt_gsa.ppost fnamtmp=[fname_,'_post']; DirectoryName = CheckPath('metropolis',M_.dname); else if options_.opt_gsa.pprior fnamtmp=[fname_,'_prior']; DirectoryName = CheckPath(['gsa' filesep 'prior'],M_.dname); else fnamtmp=[fname_,'_mc']; DirectoryName = CheckPath(['gsa' filesep 'mc'],M_.dname); end end if loadSA tmplist =load([OutDir,filesep,fnamtmp, '.mat'],'vvarvecm'); if isempty(fieldnames(tmplist)) disp('WARNING: cannot load results since the list of variables used is not present in the mat file') loadSA=0; elseif ~isequal(tmplist.vvarvecm,vvarvecm) disp('WARNING: cannot load results since the list of variables in the mat file differs from the one requested.') loadSA=0; end end if ~loadSA Y = transpose(dataset_.data); gend = dataset_.nobs; data_index = dataset_info.missing.aindex; missing_value = dataset_info.missing.state; filfilt = dir([DirectoryName filesep M_.fname '_filter_step_ahead*.mat']); filupdate = dir([DirectoryName filesep M_.fname '_update*.mat']); filparam = dir([DirectoryName filesep M_.fname '_param*.mat']); x=[]; logpo2=[]; sto_ys=[]; for j=1:length(filparam) if isempty(strmatch([M_.fname '_param_irf'],filparam(j).name)) temp=load([DirectoryName filesep filparam(j).name]); % from prior_posterior_statistics_core x=[x; temp.stock]; logpo2=[logpo2; temp.stock_logpo]; sto_ys=[sto_ys; temp.stock_ys]; clear temp; end end nruns=size(x,1); nfilt=floor(pfilt*nruns); if options_.opt_gsa.ppost || (options_.opt_gsa.ppost==0 && options_.opt_gsa.lik_only==0) skipline() disp('Computing RMSE''s...') jxj=NaN(length(vvarvecm),1); js=NaN(length(vvarvecm),1); yss=NaN(length(vvarvecm),gend,size(sto_ys,1)); for i = 1:length(vvarvecm) vj = vvarvecm{i}; jxj(i) = strmatch(vj, M_.endo_names(oo_.dr.order_var), 'exact'); js(i) = strmatch(vj, M_.endo_names, 'exact'); yss(i,:,:)=repmat(sto_ys(:,js(i))',[gend,1]); end if exist('xparam1','var') [~,~,~,ahat,~,~,aK] = DsgeSmoother(xparam1,gend,Y,data_index,missing_value,M_,oo_,options_,bayestopt_,estim_params_); y0 = reshape( squeeze(aK(1,jxj,1:gend)),[gend length(jxj)]); yobs = transpose( ahat(jxj,:)); rmse_mode = sqrt(mean((yobs(istart:end,:)-y0(istart:end,:)).^2)); r2_mode = 1-sum((yobs(istart:end,:)-y0(istart:end,:)).^2)./sum(yobs(istart:end,:).^2); end y0=-yss; %demean everything using the theoretical mean, i.e. steady state nbb=0; for j=1:length(filfilt) temp=load([DirectoryName filesep M_.fname '_filter_step_ahead',num2str(j),'.mat']); nb = size(temp.stock,4); y0(:,:,nbb+1:nbb+nb)=y0(:,:,nbb+1:nbb+nb)+reshape(temp.stock(1,js,1:gend,:),[length(js) gend nb]); nbb=nbb+nb; clear temp; end yobs=-yss; nbb=0; for j=1:length(filupdate) temp=load([DirectoryName filesep M_.fname '_update',num2str(j),'.mat']); nb = size(temp.stock,3); yobs(:,:,nbb+1:nbb+nb)=yobs(:,:,nbb+1:nbb+nb)+reshape(temp.stock(js,1:gend,:),[length(js) gend nb]); nbb=nbb+nb; clear temp; end rmse_MC=zeros(nruns,length(js)); r2_MC=zeros(nruns,length(js)); for j=1:nruns rmse_MC(j,:) = sqrt(mean((yobs(:,istart:end,j)'-y0(:,istart:end,j)').^2)); r2_MC(j,:) = 1-mean((yobs(:,istart:end,j)'-y0(:,istart:end,j)').^2)./mean((yobs(:,istart:end,j)').^2); end if exist('xparam1_mean','var') [~,~,~,ahat,~,~,aK] = DsgeSmoother(xparam1_mean,gend,Y,data_index,missing_value,M_,oo_,options_,bayestopt_,estim_params_); y0 = reshape( squeeze(aK(1,jxj,1:gend)),[gend length(jxj)]); yobs = transpose( ahat(jxj,:)); rmse_pmean = sqrt(mean((yobs(istart:end,:)-y0(istart:end,:)).^2)); r2_pmean = 1-mean((yobs(istart:end,:)-y0(istart:end,:)).^2)./mean(yobs(istart:end,:).^2); end clear stock_filter; end lnprior=NaN(nruns,1); for j=1:nruns lnprior(j,1) = priordens(x(j,:)',bayestopt_.pshape,bayestopt_.p6,bayestopt_.p7,bayestopt_.p3,bayestopt_.p4); end likelihood=logpo2(:)-lnprior(:); disp('... done!') if options_.opt_gsa.ppost save([OutDir,filesep,fnamtmp,'.mat'], 'x', 'logpo2', 'likelihood', 'rmse_MC', 'r2_MC', 'vvarvecm') if exist('xparam1_mean','var') save([OutDir,filesep,fnamtmp, '.mat'], 'rmse_pmean', 'r2_pmean','-append') end if exist('xparam1','var') save([OutDir,filesep,fnamtmp,'.mat'], 'rmse_mode', 'r2_mode','-append') end else if options_.opt_gsa.lik_only save([OutDir,filesep,fnamtmp, '.mat'], 'x', 'logpo2','likelihood', '-append') else save([OutDir,filesep,fnamtmp, '.mat'], 'x', 'logpo2','likelihood', 'rmse_MC', 'r2_MC', 'vvarvecm','-append') if exist('xparam1_mean','var') save([OutDir,filesep,fnamtmp, '.mat'], 'rmse_pmean', 'r2_pmean','-append') end if exist('xparam1','var') save([OutDir,filesep,fnamtmp,'.mat'], 'rmse_mode', 'r2_mode','-append') end end end else % loadSA if options_.opt_gsa.lik_only && options_.opt_gsa.ppost==0 load([OutDir,filesep,fnamtmp, '.mat'],'x','logpo2','likelihood'); else load([OutDir,filesep,fnamtmp, '.mat'],'x','logpo2','likelihood','rmse_MC','rmse_mode','rmse_pmean', 'r2_MC', 'vvarvecm', 'r2_mode','r2_pmean'); end lnprior=logpo2(:)-likelihood(:); nruns=size(x,1); nfilt=floor(pfilt*nruns); end % Smirnov tests nfilt0 = nfilt*ones(length(vvarvecm), 1); logpo2=logpo2(:); if ~options_.opt_gsa.ppost [~, ipost]=sort(-logpo2); [~, ilik]=sort(-likelihood); end % visual scatter analysis! if options_.opt_gsa.ppost tmp_title='R2 Scatter plot: Posterior'; atitle='R2 Scatter plot: Posterior'; asname='r2_post'; else if options_.opt_gsa.pprior tmp_title='R2 Scatter plot: Prior'; atitle='R2 Scatter plot: Prior'; asname='r2_prior'; else tmp_title='R2 Scatter plot: MC'; atitle='R2 Scatter plot: MC'; asname='r2_mc'; end end options_scatter.param_names = vvarvecm; options_scatter.param_names_tex = vvarvecm_tex; options_scatter.fname_ = fname_; options_scatter.OutputDirectoryName = OutDir; options_scatter.amcf_name = asname; options_scatter.amcf_title = atitle; options_scatter.title = tmp_title; scatter_analysis(r2_MC, x,options_scatter, options_); % end of visual scatter analysis if ~options_.opt_gsa.ppost && options_.opt_gsa.lik_only if options_.opt_gsa.pprior anam='rmse_prior_post'; atitle='RMSE prior: Log Posterior Kernel'; else anam='rmse_mc_post'; atitle='RMSE MC: Log Posterior Kernel'; end options_mcf.pvalue_ks = alpha; options_mcf.pvalue_corr = pvalue; options_mcf.alpha2 = alpha2; options_mcf.param_names = param_names; if options_.TeX options_mcf.param_names_tex = param_names_tex; else options_mcf.param_names_tex = {}; end options_mcf.fname_ = fname_; options_mcf.OutputDirectoryName = OutDir; options_mcf.amcf_name = anam; options_mcf.amcf_title = atitle; options_mcf.title = atitle; options_mcf.beha_title = 'better posterior kernel'; options_mcf.nobeha_title = 'worse posterior kernel'; if options_.TeX options_mcf.beha_title_latex = 'better posterior kernel'; options_mcf.nobeha_title_latex = 'worse posterior kernel'; end mcf_analysis(x, ipost(1:nfilt), ipost(nfilt+1:end), options_mcf, M_, options_, bayestopt_, estim_params_); if options_.opt_gsa.pprior anam = 'rmse_prior_lik'; atitle = 'RMSE prior: Log Likelihood Kernel'; else anam='rmse_mc_lik'; atitle = 'RMSE MC: Log Likelihood Kernel'; end options_mcf.amcf_name = anam; options_mcf.amcf_title = atitle; options_mcf.title = atitle; options_mcf.beha_title = 'better likelihood'; options_mcf.nobeha_title = 'worse likelihood'; if options_.TeX options_mcf.beha_title_latex = 'better likelihood'; options_mcf.nobeha_title_latex = 'worse likelihood'; end mcf_analysis(x, ilik(1:nfilt), ilik(nfilt+1:end), options_mcf, M_, options_, bayestopt_, estim_params_); else if options_.opt_gsa.ppost rmse_txt=rmse_pmean; r2_txt=r2_pmean; else if options_.opt_gsa.pprior || ~exist('rmse_pmean','var') if exist('rmse_mode','var') rmse_txt=rmse_mode; r2_txt=r2_mode; else rmse_txt=NaN(1,size(rmse_MC,2)); r2_txt=NaN(1,size(r2_MC,2)); end else rmse_txt=rmse_pmean; r2_txt=r2_pmean; end end ixx=NaN(size(rmse_MC,1),length(vvarvecm)); for i = 1:length(vvarvecm) [~, ixx(:,i)] = sort(rmse_MC(:,i)); end PP = ones(npar+nshock, length(vvarvecm)); PPV = ones(length(vvarvecm), length(vvarvecm), npar+nshock); SS = zeros(npar+nshock, length(vvarvecm)); for j = 1:npar+nshock for i = 1:length(vvarvecm) [~, P] = smirnov(x(ixx(nfilt0(i)+1:end,i),j),x(ixx(1:nfilt0(i),i),j), alpha); [H1] = smirnov(x(ixx(nfilt0(i)+1:end,i),j),x(ixx(1:nfilt0(i),i),j),alpha,1); [H2] = smirnov(x(ixx(nfilt0(i)+1:end,i),j),x(ixx(1:nfilt0(i),i),j),alpha,-1); if H1==0 && H2==0 SS(j,i)=1; elseif H1==0 SS(j,i)=-1; else SS(j,i)=0; end PP(j,i)=P; end for i = 1:length(vvarvecm) for l = 1:length(vvarvecm) if l~=i && PP(j,i)0.0001 ); vvarvecm = vvarvecm(ivar); rmse_MC = rmse_MC(:,ivar); skipline() disp(['Sample filtered the ',num2str(pfilt*100),'% best RMSE''s for each observed series ...' ]) skipline(1) title_string='RMSE ranges after filtering:'; if options_.opt_gsa.ppost==0 && options_.opt_gsa.pprior headers = {'Variable'; 'min'; 'max'; 'min'; 'max'; 'posterior mode'}; headers_tex = {'\text{Variable}'; '\text{min}'; '\text{max}'; '\text{min}'; '\text{max}'; '\text{posterior mode}'}; else headers = {'Variable'; 'min'; 'max'; 'min'; 'max'; 'posterior mean'}; headers_tex = {'\text{Variable}'; '\text{min}'; '\text{max}'; '\text{min}'; '\text{max}'; '\text{posterior mean}'}; end data_mat=NaN(length(vvarvecm),5); for j = 1:length(vvarvecm) data_mat(j,:)=[min(rmse_MC(ixx(1:nfilt0(j),j),j)) ... max(rmse_MC(ixx(1:nfilt0(j),j),j)) ... min(rmse_MC(ixx(nfilt0(j)+1:end,j),j)) ... max(rmse_MC(ixx(nfilt0(j)+1:end,j),j)) ... rmse_txt(j)]; end %get formatting for additional header line val_width = 15; val_precis = 5; label_width = max(cellofchararraymaxlength(vertcat(headers{1}, vvarvecm))+2, 0); label_format_leftbound = sprintf('%%-%ds', label_width); if all(~isfinite(data_mat)) values_length = 4; else values_length = max(ceil(max(max(log10(abs(data_mat(isfinite(data_mat))))))),1)+val_precis+1; end if any(data_mat < 0) %add one character for minus sign values_length = values_length+1; end headers_length = cellofchararraymaxlength(headers(2:end)); if ~isempty(val_width) val_width = max(max(headers_length,values_length)+2, val_width); else val_width = max(headers_length, values_length)+2; end header_string_format = sprintf('%%%ds',val_width); if options_.opt_gsa.ppost==0 && options_.opt_gsa.pprior optional_header=sprintf([label_format_leftbound,header_string_format,header_string_format,header_string_format,header_string_format],'','',['best ',num2str(pfilt*100),'% filtered'],'','remaining 90%'); else optional_header=sprintf([label_format_leftbound,header_string_format,header_string_format,header_string_format,header_string_format],'','','best filtered','','remaining'); end dyntable(options_, title_string, headers, vvarvecm, data_mat, 0, val_width, val_precis,optional_header); if options_.TeX if options_.opt_gsa.ppost==0 && options_.opt_gsa.pprior optional_header={[' & \multicolumn{2}{c}{best ',num2str(pfilt*100),' filtered} & \multicolumn{2}{c}{remaining 90\%}\\']}; else optional_header={' & \multicolumn{2}{c}{best filtered} & \multicolumn{2}{c}{remaining}\\'}; end dyn_latex_table(M_, options_, title_string, 'RMSE_ranges_after_filtering', headers_tex, vvarvecm_tex, data_mat, 0, val_width, val_precis, optional_header); end % R2 table vvarvecm=vvarvecm0; skipline() title_string='R2 over the MC sample:'; data_mat=[min(r2_MC)' max(r2_MC)']; headers = {'Variable'; 'min yr R2'; 'max yr R2'}; dyntable(options_, title_string, headers, vvarvecm, data_mat, 0, 15, 5); if options_.TeX headers_tex = {'\text{Variable}'; '\text{min yr R2}'; '\text{max yr R2}'}; dyn_latex_table(M_, options_, title_string, 'R2_MC', headers_tex, vvarvecm_tex, data_mat, 0, 15, 5); end r2_MC=r2_MC(:,ivar); vvarvecm=vvarvecm(ivar); skipline() disp(['Sample filtered the ',num2str(pfilt*100),'% best R2''s for each observed series ...' ]) skipline() disp('R2 ranges after filtering:') title_string='R2 ranges after filtering:'; if options_.opt_gsa.ppost==0 && options_.opt_gsa.pprior headers = {'Variable'; 'min'; 'max'; 'min'; 'max'; 'posterior mode'}; headers_tex = {'\text{Variable}'; '\text{min}'; '\text{max}'; '\text{min}'; '\text{max}'; '\text{posterior mode}'}; else headers = {'Variable'; 'min'; 'max'; 'min'; 'max'; 'posterior mean'}; headers_tex = {'\text{Variable}'; '\text{min}'; '\text{max}'; '\text{min}'; '\text{max}'; '\text{posterior mean}'}; end data_mat=NaN(length(vvarvecm),5); for j = 1:length(vvarvecm) data_mat(j,:)=[min(r2_MC(ixx(1:nfilt0(j),j),j)) ... max(r2_MC(ixx(1:nfilt0(j),j),j)) ... min(r2_MC(ixx(nfilt0(j)+1:end,j),j)) ... max(r2_MC(ixx(nfilt0(j)+1:end,j),j)) ... r2_txt(j)]; end %get formatting for additional header line val_width = 15; val_precis = 5; label_width = max(cellofchararraymaxlength(vertcat(headers{1}, vvarvecm))+2, 0); label_format_leftbound = sprintf('%%-%ds', label_width); if all(~isfinite(data_mat)) values_length = 4; else values_length = max(ceil(max(max(log10(abs(data_mat(isfinite(data_mat))))))),1)+val_precis+1; end if any(data_mat < 0) %add one character for minus sign values_length = values_length+1; end headers_length = cellofchararraymaxlength(headers(2:end)); if ~isempty(val_width) val_width = max(max(headers_length, values_length)+2, val_width); else val_width = max(headers_length, values_length)+2; end header_string_format = sprintf('%%%ds',val_width); if options_.opt_gsa.ppost==0 && options_.opt_gsa.pprior optional_header = sprintf([label_format_leftbound,header_string_format,header_string_format,header_string_format,header_string_format],'','',['best ',num2str(pfilt*100),'% filtered'],'','remaining 90%'); else optional_header = sprintf([label_format_leftbound,header_string_format,header_string_format,header_string_format,header_string_format],'','','best filtered','','remaining'); end dyntable(options_, title_string, headers, vvarvecm, data_mat, 0, val_width, val_precis, optional_header); if options_.TeX if ~options_.opt_gsa.ppost && options_.opt_gsa.pprior optional_header = {[' & \multicolumn{2}{c}{best ',num2str(pfilt*100),' filtered} & \multicolumn{2}{c}{remaining 90\%}\\']}; else optional_header = {' & \multicolumn{2}{c}{best filtered} & \multicolumn{2}{c}{remaining}\\'}; end dyn_latex_table(M_, options_, title_string, 'R2_ranges_after_filtering', headers_tex, vvarvecm_tex, data_mat, 0, val_width, val_precis, optional_header); end % R2 table SP = zeros(npar+nshock, length(vvarvecm)); for j = 1:length(vvarvecm) ns=find(PP(:,j)1); nsnam=(find(nsp>1)); skipline(1) disp('These parameters do not affect significantly the fit of ANY observed series:') disp(char(snam0)) skipline() disp('These parameters affect ONE single observed series:') disp(char(snam1)) skipline() disp('These parameters affect MORE THAN ONE observed series: trade off exists!') disp(char(snam2)) % plot trade-offs if ~options_.nograph a00=jet(length(vvarvecm)); if options_.opt_gsa.ppost temp_name='RMSE Posterior Tradeoffs:'; atitle='RMSE Posterior Map:'; asname='rmse_post'; else if options_.opt_gsa.pprior temp_name='RMSE Prior Tradeoffs:'; atitle='RMSE Prior Map:'; asname='rmse_prior'; else temp_name='RMSE MC Tradeoffs:'; atitle='RMSE MC Map:'; asname='rmse_mc'; end end % now I plot by observed variables options_mcf.pvalue_ks = alpha; options_mcf.pvalue_corr = pvalue; options_mcf.alpha2 = alpha2; options_mcf.param_names = param_names; options_mcf.param_names_tex = param_names_tex; options_mcf.fname_ = fname_; options_mcf.OutputDirectoryName = OutDir; for iy = 1:length(vvarvecm) options_mcf.amcf_name = [asname '_' vvarvecm{iy} '_map' ]; options_mcf.amcf_title = [atitle ' ' vvarvecm{iy}]; options_mcf.beha_title = ['better fit of ' vvarvecm{iy}]; options_mcf.nobeha_title = ['worse fit of ' vvarvecm{iy}]; if options_.TeX options_mcf.beha_title_latex = ['better fit of ' vvarvecm_tex{iy}]; options_mcf.nobeha_title_latex = ['worse fit of ' vvarvecm_tex{iy}]; end options_mcf.title = ['the fit of ' vvarvecm{iy}]; mcf_analysis(x, ixx(1:nfilt0(iy),iy), ixx(nfilt0(iy)+1:end,iy), options_mcf, M_, options_, bayestopt_, estim_params_); end for iy = 1:length(vvarvecm) ipar = find(any(squeeze(PPV(iy,:,:))