Lexical Analysis System Requirements

A lexical analyzer processes source code character streams to identify meaningful tokens and generate structured output. The system performs several core functions:

• Scans character sequences from left to right
• Identifies lexemes with semantic significance
• Produces token records (token type, actual value)
• Removes whitespace characters
• Skips comment sections
• Detects lexical errors

System Architecture:

Input: Character sequence (determined input method and storage data structure)

Processing:

• Traversal mechanism (specific scanning approach)
• Lexical rule application

Output: Token sequence (defined output format)

• Tuple representation

Token Categories:

1. Identifiers (10)
2. Unsigned numbers (11)
3. Reserved keywords (unique codes per word)
4. Operators (unique codes per operator)
5. Delimmiters (unique codes per delimiter)

Implementation Code:

#include <iostream>
#include <fstream>
#include <vector>
#include <string>
#include <cctype>

using namespace std;

// Reserved keywords array
vector<string> reserved_words = {"main", "int", "if", "else", "while", "do"};
// Corresponding category codes
vector<int> word_codes = {1, 2, 3, 4, 5, 6};
// Operators and delimiters
vector<string> operators = {"<", ">", "!=" ,">=", "<=", "==", ",", ";", "(", ")", "{", "}", "+", "-", "*", "/", "="};
// Operator category codes
vector<int> op_codes = {7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23};
// Character buffer
vector<char> char_buffer;
// Processed tokens
vector<string> token_list;
int buffer_size; // Total character count
int current_pos;

// Check if string is an operator
int check_operator(const string& token) {
    for (size_t i = 0; i < operators.size(); i++) {
        if (token == operators[i])
            return op_codes[i];
    }
    return 0;
}

// Verify numeric character
bool is_digit(char c) {
    return c >= '0' && c <= '9';
}

// Verify alphabetic character
bool is_alpha(char c) {
    return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
}

// Check if token is reserved word
int check_reserved_word(const string& token) {
    for (size_t i = 0; i < reserved_words.size(); i++) {
        if (token == reserved_words[i])
            return word_codes[i];
    }
    return 0;
}

// Determine character type
int get_char_type(char c) {
    if (is_alpha(c)) return 1; // Letter
    if (is_digit(c)) return 2; // Digit
    if (string("<>=!;,(){}+-*/").find(c) != string::npos) return 3; // Operator/delimiter
    return 0;
}

// Process identifier tokens
string process_identifier(char start_char, int pos) {
    string result(1, start_char);
    int next_pos = pos + 1;
    bool continue_scan = true;
    
    while (continue_scan && next_pos < buffer_size) {
        char current_char = char_buffer[next_pos];
        if (is_alpha(current_char) || is_digit(current_char)) {
            result += current_char;
            next_pos++;
        } else {
            continue_scan = false;
        }
    }
    current_pos = next_pos;
    return result;
}

// Process operator tokens
string process_operator(char start_char, int pos) {
    string result(1, start_char);
    int next_pos = pos + 1;
    
    if (next_pos < buffer_size) {
        char next_char = char_buffer[next_pos];
        string combined = result + next_char;
        
        // Check for multi-character operators
        if (combined == "<=" || combined == ">=" || combined == "!=" || combined == "==") {
            result = combined;
            next_pos++;
        }
    }
    current_pos = next_pos;
    return result;
}

// Process numeric tokens
string process_number(char start_char, int pos) {
    string result(1, start_char);
    int next_pos = pos + 1;
    bool continue_scan = true;
    
    while (continue_scan && next_pos < buffer_size) {
        char current_char = char_buffer[next_pos];
        if (is_digit(current_char)) {
            result += current_char;
            next_pos++;
        } else {
            continue_scan = false;
        }
    }
    current_pos = next_pos;
    return result;
}

// Output token in tuple format
void display_token(const string& token, int code) {
    cout << "(" << token << "," << code << ")" << endl;
}

// Main token extraction function
void extract_tokens() {
    for (current_pos = 0; current_pos < buffer_size;) {
        char current_char = char_buffer[current_pos];
        int char_type = get_char_type(current_char);
        
        switch (char_type) {
            case 1: { // Identifier
                string id_token = process_identifier(current_char, current_pos);
                int keyword_code = check_reserved_word(id_token);
                if (keyword_code) {
                    display_token(id_token, keyword_code);
                } else {
                    display_token(id_token, 10); // Generic identifier code
                }
                break;
            }
            case 2: { // Number
                string num_token = process_number(current_char, current_pos);
                display_token(num_token, 11); // Numeric literal code
                break;
            }
            case 3: { // Operator/delimiter
                string op_token = process_operator(current_char, current_pos);
                int op_code = check_operator(op_token);
                display_token(op_token, op_code);
                break;
            }
        }
    }
}

int main() {
    ifstream input_file("input_source.txt");
    ofstream output_file("output_tokens.txt");
    streambuf* original_cout = cout.rdbuf();
    cout.rdbuf(output_file.rdbuf());
    
    buffer_size = 0;
    char ch;
    
    // Read input file, skipping spaces
    while (input_file.get(ch)) {
        if (ch != ' ' && ch != '\t' && ch != '\n') {
            char_buffer.push_back(ch);
            buffer_size++;
        }
    }
    
    extract_tokens();
    
    input_file.close();
    output_file.close();
    cout.rdbuf(original_cout);
    return 0;
}

Sample Input File (input_source.txt):

int main()
{
    int m,n;
    int num=0;
    n=10;
    if(n<0){
       m=-n;
    }
    else{
       m=n;
    }
    while(m!=0){
             num=m*n;
    }
}

Expected Output:

(int,2)
(main,1)
((,15)
(),16)
({,17)
(int,2)
(m,10)
(,,13)
(n,10)
(;,14)
(int,2)
(num,10)
(=,23)
(0,11)
(;,14)
(n,10)
(=,23)
(10,11)
(;,14)
(if,3)
((,15)
(n,10)
(<,7)
(0,11)
(),16)
({,17)
(m,10)
(=,23)
(-,20)
(n,10)
(;,14)
(},18)
(else,4)
({,17)
(m,10)
(=,23)
(n,10)
(;,14)
(},18)
(while,5)
((,15)
(m,10)
(!=,9)
(0,11)
(),16)
({,17)
(num,10)
(=,23)
(m,10)
(*,21)
(n,10)
(;,14)
(},18)
(},18)