54 lines
1.6 KiB
Matlab
Executable File
54 lines
1.6 KiB
Matlab
Executable File
function tokens = tokenize(source,delimiter)
|
|
% syntax
|
|
% tokens = tokenize(source,delimiters)
|
|
%
|
|
% source is a string to be broken into tokens
|
|
% delimiters is a character array of single character delimiters
|
|
% tokens is a cell string array containing the tokens
|
|
|
|
|
|
posdelims = [];
|
|
|
|
% assumes that delimiter cannot be in the first position or the last
|
|
% position
|
|
ndelimiters = size(delimiter,1);
|
|
for i=1:ndelimiters
|
|
newpositions = strfind(source,delimiter(i,:));
|
|
if ~isempty(newpositions)
|
|
posdelims =[posdelims, newpositions];
|
|
end
|
|
end
|
|
|
|
% reorder posdelims in ascending order
|
|
posdelims = sort(posdelims);
|
|
|
|
if isempty(posdelims)
|
|
tokens = cellstr(source);
|
|
else
|
|
ndelims = length(posdelims);
|
|
% build positions for substrings
|
|
delims = zeros(ndelims+1,2);
|
|
for i=1:ndelims+1
|
|
if i==1
|
|
if posdelims(1) == 1
|
|
tokens = cellstr(source(1));
|
|
else
|
|
delims(i,:) = [1,posdelims(i)-1];
|
|
tokens = cellstr(source([delims(i,1):delims(i,2)]));
|
|
tokens = [tokens, source(posdelims(i))];
|
|
end
|
|
elseif i==ndelims+1
|
|
if (posdelims(i-1) < length(source))
|
|
delims(i,:) = [posdelims(i-1)+1,length(source)];
|
|
tokens = [tokens, cellstr(source([delims(i,1):delims(i,2)]))];
|
|
end
|
|
else
|
|
if posdelims(i)>posdelims(i-1)+1
|
|
delims(i,:) = [posdelims(i-1)+1,posdelims(i)-1];
|
|
tokens = [tokens, cellstr(source([delims(i,1):delims(i,2)]))];
|
|
end
|
|
tokens = [tokens, source(posdelims(i))];
|
|
end
|
|
end
|
|
|
|
end |