New utilities for runtime transfer of files from remote slaves.
Marco Ratto
parent
a4b66e63f1
commit
3d80322303
|
|
@ -0,0 +1,77 @@
|
|||
function [NewFilesFrom, NewFileList]=dynareParallelFindNewFiles(FileList,Parallel, PRCDir)
|
||||
|
||||
% PARALLEL CONTEXT
|
||||
% In a parallel context, this function checks if in 'dirName' and its sub-directory
|
||||
% there are other files in addition to 'FileList'.
|
||||
% If 'Yes' the variable 'NewFiles' is a list of these file.
|
||||
%
|
||||
%
|
||||
% INPUTS
|
||||
% o dirName [] ...
|
||||
% o FileList [] ...
|
||||
%
|
||||
% OUTPUTS
|
||||
% o NewFilesFromSlaves [] ...
|
||||
%
|
||||
% Copyright (C) 2009-2011 Dynare Team
|
||||
%
|
||||
% This file is part of Dynare.
|
||||
%
|
||||
% Dynare is free software: you can redistribute it and/or modify
|
||||
% it under the terms of the GNU General Public License as published by
|
||||
% the Free Software Foundation, either version 3 of the License, or
|
||||
% (at your option) any later version.
|
||||
%
|
||||
% Dynare is distributed in the hope that it will be useful,
|
||||
% but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
% GNU General Public License for more details.
|
||||
%
|
||||
% You should have received a copy of the GNU General Public License
|
||||
% along with Dynare. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
NewFilesFrom={};
|
||||
|
||||
LfL=length(FileList);
|
||||
% The first call ...
|
||||
NewFileList = dynareParallelListAllFiles('Root',PRCDir,Parallel);
|
||||
|
||||
|
||||
LnFl=length(NewFileList);
|
||||
|
||||
RelativePosition=1;
|
||||
|
||||
for i=1:LnFl
|
||||
|
||||
% Exception Handling
|
||||
|
||||
% If you comment the code below all new file will be copied!
|
||||
|
||||
% 1. The comp_status* files are managed separately.
|
||||
|
||||
FiCoS=strfind(NewFileList{i},'comp_status_');
|
||||
if ~isempty(FiCoS)
|
||||
continue
|
||||
end
|
||||
|
||||
% 2. For the time being is not necessary to always copy *.log
|
||||
% and *.txt files.
|
||||
|
||||
nC1 = strfind(NewFileList{i}, '.log');
|
||||
nC2 = strfind(NewFileList{i}, '.txt');
|
||||
|
||||
if (~isempty(nC1) || ~isempty(nC2))
|
||||
continue
|
||||
end
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
|
||||
|
||||
N=strmatch(NewFileList{i},FileList,'exact');
|
||||
if isempty(N)
|
||||
NewFilesFrom{RelativePosition}=NewFileList{i};
|
||||
RelativePosition=RelativePosition+1;
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
function [PRCDirSnapshot]=dynareParallelGetNewFiles(PRCDir,Parallel,PRCDirSnapshot)
|
||||
% PARALLEL CONTEXT
|
||||
% In a parallel context, this is a specialized function able to ...
|
||||
%
|
||||
%
|
||||
% INPUTS
|
||||
% o NamFileInput [] ...
|
||||
% o PRCDir [] ...
|
||||
% o Parallel [] ...
|
||||
%
|
||||
% OUTPUTS
|
||||
% None
|
||||
%
|
||||
%
|
||||
%
|
||||
% Copyright (C) 2009-2011 Dynare Team
|
||||
%
|
||||
% This file is part of Dynare.
|
||||
%
|
||||
% Dynare is free software: you can redistribute it and/or modify
|
||||
% it under the terms of the GNU General Public License as published by
|
||||
% the Free Software Foundation, either version 3 of the License, or
|
||||
% (at your option) any later version.
|
||||
%
|
||||
% Dynare is distributed in the hope that it will be useful,
|
||||
% but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
% GNU General Public License for more details.
|
||||
%
|
||||
% You should have received a copy of the GNU General Public License
|
||||
% along with Dynare. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
NewFilesFromSlaves={};
|
||||
|
||||
% try
|
||||
for indPC=1:length(Parallel),
|
||||
|
||||
if Parallel(indPC).Local==0;
|
||||
[NewFilesFromSlaves, PRCDirSnapshot{indPC}]=dynareParallelFindNewFiles(PRCDirSnapshot{indPC},Parallel(indPC), PRCDir);
|
||||
if ~ispc || strcmpi('unix',Parallel(indPC).OperatingSystem),
|
||||
fS='/';
|
||||
else
|
||||
fS='\';
|
||||
end
|
||||
|
||||
if ~isempty(NewFilesFromSlaves)
|
||||
|
||||
for i=1:length(NewFilesFromSlaves)
|
||||
SlashNumberAndPosition=[];
|
||||
PRCDirPosition=findstr(NewFilesFromSlaves{i}, ([PRCDir]));
|
||||
sT=NewFilesFromSlaves{i};
|
||||
sT(1:(PRCDirPosition+length([PRCDir]))-2)=[];
|
||||
sT(1)='.';
|
||||
SlashNumberAndPosition=findstr(sT,fS);
|
||||
fileaddress={sT(1:SlashNumberAndPosition(end)),sT(SlashNumberAndPosition(end)+1:end)};
|
||||
dynareParallelGetFiles(fileaddress,PRCDir,Parallel(indPC));
|
||||
|
||||
display('NuovoFilecopiato -->');
|
||||
display(fileaddress{2});
|
||||
display('<--');
|
||||
|
||||
end
|
||||
else
|
||||
continue
|
||||
end
|
||||
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,110 @@
|
|||
function fileList = dynareParallelListAllFiles(dirName,PRCDir,Parallel)
|
||||
|
||||
% PARALLEL CONTEXT
|
||||
% In a parallel context, this function searches recursively through all subdirectories
|
||||
% of the given directory 'dirName' and then return a list of all file
|
||||
% finds in 'dirName'.
|
||||
%
|
||||
%
|
||||
% INPUTS
|
||||
% o PRCDir [] ...
|
||||
% o Parallel [] ...
|
||||
%
|
||||
% OUTPUTS
|
||||
% o fileList [] ...
|
||||
%
|
||||
% Copyright (C) 2009-2011 Dynare Team
|
||||
%
|
||||
% This file is part of Dynare.
|
||||
%
|
||||
% Dynare is free software: you can redistribute it and/or modify
|
||||
% it under the terms of the GNU General Public License as published by
|
||||
% the Free Software Foundation, either version 3 of the License, or
|
||||
% (at your option) any later version.
|
||||
%
|
||||
% Dynare is distributed in the hope that it will be useful,
|
||||
% but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
% GNU General Public License for more details.
|
||||
%
|
||||
% You should have received a copy of the GNU General Public License
|
||||
% along with Dynare. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
|
||||
if (~ispc || strcmpi('unix',Parallel.OperatingSystem))
|
||||
|
||||
fileList={};
|
||||
currentPath=[];
|
||||
|
||||
% Get the data for the current remote directory.
|
||||
[Flag fL]=system(['ssh ',Parallel.UserName,'@',Parallel.ComputerName,' ls ',Parallel.RemoteDirectory,'/',PRCDir, ' -R -p -1']);
|
||||
|
||||
% Format the return value fL.
|
||||
|
||||
nL=regexp(fL,'\n');
|
||||
start=1;
|
||||
j=1;
|
||||
|
||||
for (i=1:length(nL))
|
||||
|
||||
stringProcessed=fL(start:nL(i)-1);
|
||||
|
||||
if isempty(stringProcessed)
|
||||
start=nL(i)+1;
|
||||
continue
|
||||
end
|
||||
|
||||
if strfind(stringProcessed,'/')
|
||||
if strfind(stringProcessed,PRCDir)
|
||||
DD=strfind(stringProcessed,':');
|
||||
stringProcessed(DD)='/';
|
||||
currentPath=stringProcessed;
|
||||
end
|
||||
start=nL(i)+1;
|
||||
continue
|
||||
end
|
||||
|
||||
fileList{j,1}=[currentPath stringProcessed];
|
||||
start=nL(i)+1;
|
||||
j=j+1;
|
||||
end
|
||||
|
||||
|
||||
else
|
||||
if (strmatch(dirName, 'Root')==1) % First call in Windows enviroment
|
||||
dirName=(['\\',Parallel.ComputerName,'\',Parallel.RemoteDrive,'$\',Parallel.RemoteDirectory,'\',PRCDir]);
|
||||
end
|
||||
% Get the data for the current directory and exstract from it the index
|
||||
% for directories:
|
||||
dirData = dir(dirName);
|
||||
dirIndex = [dirData.isdir];
|
||||
|
||||
% Get a list of the files:
|
||||
fileList = {dirData(~dirIndex).name}';
|
||||
|
||||
% Build the path files:
|
||||
if ~isempty(fileList)
|
||||
fileList = cellfun(@(x) fullfile(dirName,x),...
|
||||
fileList,'UniformOutput',false);
|
||||
end
|
||||
|
||||
% Get a list of the subdirectories:
|
||||
subDirs = {dirData(dirIndex).name};
|
||||
|
||||
% Find index of subdirectories that are not '.' or '..':
|
||||
validIndex = ~ismember(subDirs,{'.','..'});
|
||||
|
||||
% Loop over valid subdirectories (i.e. all subdirectory without '.' and
|
||||
% '..':
|
||||
|
||||
for iDir = find(validIndex)
|
||||
% Get the subdirectory path:
|
||||
nextDir = fullfile(dirName,subDirs{iDir});
|
||||
|
||||
% Recursively call dynareParallelListAllFiles:
|
||||
fileList = [fileList; dynareParallelListAllFiles(nextDir,PRCDir,Parallel)];
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
function [PRCDirSnapshot]=dynareParallelSnapshot(PRCDir,Parallel)
|
||||
% PARALLEL CONTEXT
|
||||
% In a parallel context, this simply record the directory's files at time
|
||||
% 't0'.
|
||||
%
|
||||
%
|
||||
% INPUTS
|
||||
% o [] ...
|
||||
% o [] ...
|
||||
% o [] ...
|
||||
%
|
||||
% OUTPUTS
|
||||
% o [] ...
|
||||
%
|
||||
%
|
||||
%
|
||||
% Copyright (C) 2009-2011 Dynare Team
|
||||
%
|
||||
% This file is part of Dynare.
|
||||
%
|
||||
% Dynare is free software: you can redistribute it and/or modify
|
||||
% it under the terms of the GNU General Public License as published by
|
||||
% the Free Software Foundation, either version 3 of the License, or
|
||||
% (at your option) any later version.
|
||||
%
|
||||
% Dynare is distributed in the hope that it will be useful,
|
||||
% but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
% GNU General Public License for more details.
|
||||
%
|
||||
% You should have received a copy of the GNU General Public License
|
||||
% along with Dynare. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
PRCDirSnapshot={};
|
||||
|
||||
for indPC=1:length(Parallel),
|
||||
if Parallel(indPC).Local==0;
|
||||
% The first call ...
|
||||
PRCDirSnapshot{indPC}=dynareParallelListAllFiles('Root',PRCDir,Parallel(indPC));
|
||||
|
||||
end
|
||||
end
|
||||
Loading…
Reference in New Issue