function tokens = tokenize(source,delimiter)
% syntax
% tokens = tokenize(source,delimiters)
%
% source is a string to be broken into tokens
% delimiters is a character array of single character delimiters
% tokens is a cell string array containing the tokens


posdelims = [];

% assumes that delimiter cannot be in the first position or the last
% position
ndelimiters = size(delimiter,1);
for i=1:ndelimiters
    newpositions = strfind(source,delimiter(i,:));
    if ~isempty(newpositions)
        posdelims =[posdelims, newpositions];
    end
end

% reorder posdelims in ascending order
posdelims = sort(posdelims);

if isempty(posdelims)
    tokens = cellstr(source);
else
    ndelims = length(posdelims);
    % build positions for substrings
    delims = zeros(ndelims+1,2);
    for i=1:ndelims+1
        if i==1
            if posdelims(1) == 1
                tokens = cellstr(source(1));
            else
                delims(i,:) = [1,posdelims(i)-1];
                tokens = cellstr(source([delims(i,1):delims(i,2)]));
                tokens = [tokens, source(posdelims(i))];
            end
        elseif  i==ndelims+1
            if (posdelims(i-1) < length(source))
                delims(i,:) = [posdelims(i-1)+1,length(source)];
                tokens = [tokens, cellstr(source([delims(i,1):delims(i,2)]))];
            end
        else
            if posdelims(i)>posdelims(i-1)+1
                delims(i,:) = [posdelims(i-1)+1,posdelims(i)-1];
                tokens = [tokens, cellstr(source([delims(i,1):delims(i,2)]))];
            end
            tokens = [tokens, source(posdelims(i))];
        end
    end

end