function [] = read_ngc(branch_data,name_data);
%
% function [] = read_ngc(branch_data,name_data);
%
%  N. Taylor,  November 2002.
%
% Reads data from the format of the spreadsheets (XLS) used in the National Grid Company's 
% Seven Year Statement appendices.  The relevant chunks of data should be pasted into a text file, 
% with tab separators between columns and line-feed characters between rows (these will be there
% naturally if copied and pasted directly from, say, OpenOffice to a text editor.)  It may be 
% necessary to remove a few 'commenty' lines from the spreadsheet format.
%
% The following FORMATS are assumed:
%
% INPUT:
%
%    branch_data  - filename (with extension) as a string, of branch data text file:  the file should
%               have the following format  (tab separated values, one line for each described branch): -
%
%            frombus_code   tobus_code  line_length  line_type  R  X  B  wint aut summ spring
%
%       where R X B are % on 100MVA Vn bases, and wint - spring are seasonal max MVA loadings.
%
%       After the four-letter station code of the bus names, the fifth digit is 4 for 400 kV, 
%       2 for 275 kV, 1 for 132 kV, 6 for 66 kV, 3 for 33 kV.  The sixth digit refers to the 
%       sectioning of buses in the station.
%
%
%    name_data   - filename (with extension) as a string, of station naming data file  (a station will 
%               probably have more than one network node):  the file should
%               have the following format (tab separated values, one line for each station-name): -  
%
%            code   full-name
%
%       where  code  is the four (capital) letter code, as used in the line data,  and full-name is 
%       the full name with possible space characters.
%
% OUTPUT  (as global variables):
%
%      all matrices and arrays have row numbers corresponding to the order of branch data in
%      the original input file.  The structure of each output is as described below...
%
%      nstat     number of station code with translations given (in file 'name_data')
%      nbranch   number of branches given in the system data (in file 'branch_data')
%      nbus      number of distinct buses found in the line to and from data
%   
%      statnames  {nstat,2}, structure with four-letter station codes in first column and 
%                            full names in second column.  Ordering is alphabetical.
%
%      busnames   {nbus,2}, structure with six-letter codes in first column, and full 
%                           descriptions in the second.  Ordering corresponds to the 
%                           arbitrary bus-numbers assigned in the line to and from 
%                           data in 'connect'.
%
%      connect  (nbranch,2), matrix of to- and from-bus numbers, the numbers corresponding to indices into 'busnames'
%      elec     (nbranch,3), matrix of R X B pu
%      rate     (nbranch,4), matrix of max MVA loadings for each season for each branch (spring, summer, autumn, winter)
%      type     {nbranch,1}, structure of strings describing the branch type
%      leng     (nbranch,1), matrix of branch lengths in km (0m for transformer, reactor, etc)
%   

global nbranch nstat nbus statnames busnames connect elec rate type leng perm

jay = sqrt(-1);

%%%%%%%%%%%%%%%%%%%%% Read branch_data and name_data files, and put their lines into data structures %%%%%%%%%%%%%%%%%%%%%%%%

for loopcounter = 1:2,
    if loopcounter==1; name = branch_data; end;
    if loopcounter==2; name = name_data; end;
    
    % Read-in file as a continuous string of characters, including line-end characters ..

    % .. method requiring UNIX command (system dependent)
    % [dummy,txt] = unix(['cat ', name]);
    % n_char_input = length(txt);
     
    % .. method using Matlab's own functions ...
    text = [];
    fileID = fopen(name);
    while 1,
        % take a line without newline character
        tline = fgetl(fileID);
        % if at EOF, remove any leading newline and put one at end..
        if ~ischar(tline),     
            tempfin = length(text);
            if text(1,1)==sprintf('\n'),
                tempstt = 2;
            else
                tempstt = 1;
            end
            text = [text(1,tempstt:tempfin), sprintf('\n')];
            break
        end
        text = [text, sprintf('\n'), tline];
    end
    fclose(fileID);
    n_char_input = length(text);
    
    % mark positions of line-ends, and add an initial one _before_ the start of the first line,
    % so that the following processing can treat all lines in the same way.
    line_ends = find(text==sprintf('\n'));
    n_line_ends = length(line_ends);
    
    % check for file data that will cause errors in file-reading...
    if n_line_ends <= 7,
        if n_line_ends<=1,
            warning(' The result of cat''ing the input file into a variable has been an empty... is the filename invalid?')
        else
            warning([' The file ',name,' has so few end-of line characters that it is either not a  ',sprintf('\n'),...
                    ' sufficient load-flow data file, or else has incompatible end-of-line characters for',sprintf('\n'), ...
                    ' this conversion method.  Please check the file content and formatting. '])
        end
        disp(' ')
    end
    
    % check for strangeness at the end of the file (no final line-end)
    if line_ends(n_line_ends) ~= n_char_input,
        warning([' Note:  the file ',name,' does not end with an end-of-line character.'])
    end
    
    % enforce single EOL characters, by the replacing all but the last of consecutive EOLs
    % by a single space character per EOL character.
    for count = 2:n_line_ends,
        if line_ends(count) == line_ends(count-1)+1,
            text(line_ends(count-1)) = ' ';
        end
    end
    
    % recalculate the number of line-ends
    line_ends = find(text==sprintf('\n'));
    % insert end point of the imaginary line before the first real one ...
    line_ends = [0, line_ends];
    n_line_ends = length(line_ends);
    
    % take each line, and save as a string without EOL character, in a data-structure indexed by line numbers...
    % The structure "store_lines" has line strings in the second column, and descriptions in the first.
    temp = [];
    for count = 1:n_line_ends-1
        charstring = text( line_ends(count)+1 : line_ends(count+1)-1 );
        temp{count,1} = charstring;
        if loopcounter == 1,
            branchstrc = temp;
        elseif loopcounter == 2,
            namestrc = temp;
        end
    end
    
end % end the 'for name = branch_data or name_data'

nbranch = length(branchstrc);
nstat = length(namestrc);

% clean up.  Now branchcstrc and namestrc are structures of a text line in each element.
clear loopcounter name temp count charstring line_ends text dummy n_line_ends n_char_input

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Interpret the data in  namestrc  and  branchstrc  to make matrices and structures from it.
% Note that there are more bus codes than statio codes - a bus code is a station code with
% two extra characters appended, showing voltage and bus number within the station.  
% The number of buses and their names can only be given after exhasutive search of the
% to- and from-buses in the branch data.

% for the namestrc structure, holding code names and full names, split each line into the two strings,
for i=1:nstat,
    % take current line
    temp = namestrc{i,1};
    % assume that the first four characters are the code name, 
    statnames{i,1} = temp(1,1:4);
    % now process just the remainder ...
    temp = temp(1,5:length(temp));
    % find all whitespace
    indx = find(temp==sprintf('\t') | temp==' ');
    % confirm first character being whitespace
    if isempty(indx) | indx(1)~=1,
        warning([' Line ',num2str(i),' of the name data does not have whitespace in column 5.'])
    else
        if ~isempty(indx),
            while indx(1)==1,
                % remove initial whitespace
                temp = temp(1,2:length(temp));
                indx = find(temp==sprintf('\t') | temp==' ');
                if isempty(indx); break; end
            end
        end
        if ~isempty(indx),
            while indx(length(indx))==length(temp),
                % remove final whitespace
                temp = temp(1:length(temp)-1);
                indx = find(temp==sprintf('\t') | temp==' ');
                if isempty(indx); break; end
            end
        end
    end
    % now 'statnames' has the first element in each row holding the code for 
    % that station name, and the second element holding the full name.
    statnames{i,2} = temp(1,1:length(temp)); 
end
clear  i temp indx

% process branch data
for i=1:nbranch,
    
    % take current line
    temp = branchstrc{i};
    % find where the tab separaters are
    indx = find(temp == sprintf('\t'));
    exptab = 10;  % expected number of tabs
    if length(indx)~=exptab,
        disp(['Line ',num2str(i),' of the data contains ',num2str(length(indx)),' tabs, rather than the expected ',num2str(exptab),' tabs.'])
    end
    clear exptab
    
    % Reminder of the structure of each line of the inpnut file:
    %     frombus_code   tobus_code  line_length  line_type   R%   X%   B%   wint  aut  summ  spring
    
    % convert the ready-matricised electrical data to number format, R X B ...
    elec(i,:) = str2num(temp( indx(4) : indx(7) )) /100 ;
    
    % convert to numbers, and permutate, the loading data (MVA:  Spring, Summer, Autumn, Winter)
    rate(i,:) = [ str2num(temp(indx(10):length(temp)))  str2num(temp(indx(9):indx(10)))  ...
            str2num(temp(indx(8):indx(9)))        str2num(temp(indx(7):indx(8))) ];
    
    % insert the km line-lengths -  replace N/A with 0
    % if the branch is a transformer or reactor....
    if isempty(strfind(temp(indx(2):indx(3)),'N/A')),
        leng(i,1) = str2num(temp( indx(2) : indx(3) ));
    else
        leng(i,1) = 0;
    end
    
    % branch type is a text descriptor
    type{i,1} = (temp( indx(3)+1 : indx(4)-1 ));
    
    % the bus-codes can be read straight off...
    code{i,1} = (temp( 1 : indx(1)-1 ));
    code{i,2} = (temp( indx(1)+1 : indx(2)-1 ));
    
end
clear  i temp indx

% Take the structure ('code') of to and for buses for various lines, and replace
% each bus-name with an arbitrary identifier number, for ease of manipulation.
% The structure "list" contains names with structure position of their assigned
% numbers.  The matrix "connect" is the original structure "code", but with 
% names replaced by the assigned numbers.
% The ordering may be changed later (in another function) to make electrically 
% close buses close in the ordering too.
list = []; connect = [];
list{1,1} = code{1,1}; % include first point, to ease later comparison...
connect(1,1) = 1; % the arbitrary number of the first point is 1
for i=1:nbranch,
    for j=1:2,
        temp = code{i,j}; % take the next bus name
        for k=1:length(list),
            if strcmp(temp,list{k,1}),
                assignednumber = k;
                alreadythere = 1;
                break
            else
                assignednumber = length(list)+1; 
                alreadythere=0;
            end
        end            
        connect(i,j) = assignednumber;
        if alreadythere==0,
            list{assignednumber,1} = temp; % insert in the structure of names
        end
    end
end

nbus = length(list);

% Display the number of buses found...
disp(' '); disp([' ',num2str(nbus),' distinct buses have been found by read_ngc.m ']); disp(' ');


% Now, make a new, two column, structure with the bus-code list in the first column
% and the full descriptor of the bus in the second.  This structure replaces 'list'.
fullnames = [];
for i=1:nbus,
    
    % take the last two of the six characters from the bus code, for from and to buses
    temp = list{i,1}(1,5:6);
    % for the from and then to, interpret the first of the two descriptor codes
    if     temp(1)=='6';  str = '66kV';
    elseif temp(1)=='4';  str = '400kV';
    elseif temp(1)=='3';  str = '33kV';
    elseif temp(1)=='2';  str = '275kV';
    elseif temp(1)=='1';  str = '132kV';
    end
    % find the station name in 'statnames', and concatenate with voltage and bus-number
    for j=1:nstat,
        if ~isempty( strfind(list{i,1},statnames{j,1}) ),
            fullnames{i,1} = [statnames{j,2}, ', ',str, ', bus ', temp(2)];
            break
        end
    end   
    % warn if the station code is not found in 'statnames'
    if isempty(fullnames{i,1}),
        fullnames{i,1} = '** Full name details not found **';
        warning([' No reference found in full name data to station-code  ',list{i,1}(1:4),])
    end
    
end

busnames = [list, fullnames];

% clear-up workspace...
clear i j  temp assignednumber alreadythere str
clear code list fullnames namestrc branchstrc 


%  Order the bus numbers, to make the Ybus matrix vaguely diagonal
% (physically close buses to be close in the ordering system)

% Make a matrix describing pure topological connectivity of the system (1 or 0) 
% and use the Matlab SYMRCM function to arrange it to be close to diagonal.  
% This is simple to use, works well, but doesn't allow for sub-ranking by admittance.
conns = zeros(nbus,nbus); 
% conns is made to have one for a connection, 0 otherwise
for i=1:length(connect),
    conns(connect(i,1),connect(i,2))=1; 
    conns(connect(i,2),connect(i,1))=1; 
end

global perm
% Choose type of solution, and call the cool function 'order_ngc'
% to produce the permutation vector, 'perm'.
method = input(' Enter ordering method: 0 for SYMRCM, 1 for TODAY''S SPECIAL > ');
startpos = 'HARK40';
% perm = OLD_order_ngc(startpos, method);
perm = OLD(startpos, method);

% Permutate each vector of buses:
busnames = busnames(perm,:);

% Alter numbers in the connect matrix, to match the permutated bus order
connectnew = zeros(nbranch,2);
for i=1:nbranch,
    for j=1:2
        connectnew(i,j) = find(connect(i,j)==perm);
    end
end
connect = connectnew;