Lexical Analysis Fundamentals

Lexical analysis serves as the initial processing phase in compilers and interpreters, transforming source code strings into structured tokens. These tokens provide the foundation for subsequent parsing and execution phases.

The lexical analyzer performs several critical functions:

• Scanning source text character by character
• Identifying fundamental elemenst including words, numbers, symbols, and strings
• Filtering out whitespace and comments
• Encapsulating recognized elements as tokens with type, literal value, and line number information

Implementing the Lexer

Token Type Definition

public enum SymbolType
{
    // Single character symbols
    LEFT_PARENTHESIS, RIGHT_PARENTHESIS, LEFT_CURLY, RIGHT_CURLY,
    COMMA, PERIOD, HYPHEN, PLUS, SEMICOLON, FORWARD_SLASH, ASTERISK,

    // Operators (one or two characters)
    EXCLAMATION, EXCLAMATION_EQUALS,
    ASSIGNMENT, EQUALITY,
    GREATER_THAN, GREATER_EQUAL,
    LESS_THAN, LESS_EQUAL,

    // Literals
    NAME, TEXT_LITERAL, NUMERIC_LITERAL,

    // Reserved words
    AND, CLASS, ELSE, FALSE, FUNCTION, FOR, IF, NULL, OR,
    PRINT, RETURN, SUPER, THIS, TRUE, VARIABLE, WHILE,

    // End marker
    END_OF_FILE
}

public class Lexeme
{
    public SymbolType Category { get; }
    public string Content { get; }
    public object Value { get; }
    public int LineNumber { get; }

    public Lexeme(SymbolType category, string content, object value, int lineNumber)
    {
        Category = category;
        Content = content;
        Value = value;
        LineNumber = lineNumber;
    }

    public override string ToString()
    {
        return $"{Category} {Content} {Value}";
    }
}

Lexical Scanner Implementation

public class LexicalProcessor
{
    private readonly string inputText;
    private readonly List<lexeme> lexemes = new();

    private int scanStart = 0;
    private int scanPosition = 0;
    private int currentLine = 1;

    private static readonly Dictionary<string symboltype=""> reservedWords = new()
    {
        { "and", SymbolType.AND }, { "class", SymbolType.CLASS },
        { "else", SymbolType.ELSE }, { "false", SymbolType.FALSE },
        { "for", SymbolType.FOR }, { "fun", SymbolType.FUNCTION },
        { "if", SymbolType.IF }, { "nil", SymbolType.NULL },
        { "or", SymbolType.OR }, { "print", SymbolType.PRINT },
        { "return", SymbolType.RETURN }, { "super", SymbolType.SUPER },
        { "this", SymbolType.THIS }, { "true", SymbolType.TRUE },
        { "var", SymbolType.VARIABLE }, { "while", SymbolType.WHILE }
    };

    public LexicalProcessor(string inputText)
    {
        this.inputText = inputText;
    }

    public List<lexeme> ProcessTokens()
    {
        while (!ReachedEnd())
        {
            scanStart = scanPosition;
            ProcessNextToken();
        }

        lexemes.Add(new Lexeme(SymbolType.END_OF_FILE, "", null, currentLine));
        return lexemes;
    }

    private bool ReachedEnd() => scanPosition >= inputText.Length;

    private void ProcessNextToken()
    {
        char currentChar = ReadNextChar();

        switch (currentChar)
        {
            case '(': AddLexeme(SymbolType.LEFT_PARENTHESIS); break;
            case ')': AddLexeme(SymbolType.RIGHT_PARENTHESIS); break;
            case '{': AddLexeme(SymbolType.LEFT_CURLY); break;
            case '}': AddLexeme(SymbolType.RIGHT_CURLY); break;
            case ',': AddLexeme(SymbolType.COMMA); break;
            case '.': AddLexeme(SymbolType.PERIOD); break;
            case '-': AddLexeme(SymbolType.HYPHEN); break;
            case '+': AddLexeme(SymbolType.PLUS); break;
            case ';': AddLexeme(SymbolType.SEMICOLON); break;
            case '*': AddLexeme(SymbolType.ASTERISK); break;
            case '!': AddLexeme(CheckNext('=') ? SymbolType.EXCLAMATION_EQUALS : SymbolType.EXCLAMATION); break;
            case '=': AddLexeme(CheckNext('=') ? SymbolType.EQUALITY : SymbolType.ASSIGNMENT); break;
            case '<': AddLexeme(CheckNext('=') ? SymbolType.LESS_EQUAL : SymbolType.LESS_THAN); break;
            case '>': AddLexeme(CheckNext('=') ? SymbolType.GREATER_EQUAL : SymbolType.GREATER_THAN); break;
            case '/':
                if (CheckNext('/'))
                {
                    while (PreviewChar() != '\n' && !ReachedEnd()) ReadNextChar();
                }
                else
                {
                    AddLexeme(SymbolType.FORWARD_SLASH);
                }
                break;
            case ' ': case '\r': case '\t': break;
            case '\n': currentLine++; break;
            case '"': ProcessString(); break;
            default:
                if (IsNumeric(currentChar)) ProcessNumber();
                else if (IsAlphabetic(currentChar)) ProcessIdentifier();
                else Console.WriteLine($"[Line {currentLine}] Unexpected character: {currentChar}");
                break;
        }
    }

    private char ReadNextChar() => inputText[scanPosition++];

    private void AddLexeme(SymbolType category) => AddLexeme(category, null);

    private void AddLexeme(SymbolType category, object value)
    {
        string textContent = inputText.Substring(scanStart, scanPosition - scanStart);
        lexemes.Add(new Lexeme(category, textContent, value, currentLine));
    }

    private bool CheckNext(char expectedChar)
    {
        if (ReachedEnd()) return false;
        if (inputText[scanPosition] != expectedChar) return false;
        scanPosition++;
        return true;
    }

    private char PreviewChar() => ReachedEnd() ? '\0' : inputText[scanPosition];

    private char PreviewNextChar()
    {
        if (scanPosition + 1 >= inputText.Length) return '\0';
        return inputText[scanPosition + 1];
    }

    private void ProcessString()
    {
        while (PreviewChar() != '"' && !ReachedEnd())
        {
            if (PreviewChar() == '\n') currentLine++;
            ReadNextChar();
        }

        if (ReachedEnd())
        {
            Console.WriteLine($"[Line {currentLine}] Unterminated string.");
            return;
        }

        ReadNextChar(); // Closing quote
        string stringValue = inputText.Substring(scanStart + 1, scanPosition - scanStart - 2);
        AddLexeme(SymbolType.TEXT_LITERAL, stringValue);
    }

    private void ProcessNumber()
    {
        while (IsNumeric(PreviewChar())) ReadNextChar();

        if (PreviewChar() == '.' && IsNumeric(PreviewNextChar()))
        {
            ReadNextChar();
            while (IsNumeric(PreviewChar())) ReadNextChar();
        }

        double numericValue = double.Parse(inputText.Substring(scanStart, scanPosition - scanStart));
        AddLexeme(SymbolType.NUMERIC_LITERAL, numericValue);
    }

    private void ProcessIdentifier()
    {
        while (IsAlphaNumeric(PreviewChar())) ReadNextChar();

        string identifierText = inputText.Substring(scanStart, scanPosition - scanStart);
        SymbolType category = reservedWords.TryGetValue(identifierText, out SymbolType type) ? type : SymbolType.NAME;
        AddLexeme(category);
    }

    private bool IsNumeric(char c) => c >= '0' && c <= '9';
    private bool IsAlphabetic(char c) => (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_';
    private bool IsAlphaNumeric(char c) => IsAlphabetic(c) || IsNumeric(c);
}
</lexeme></string></lexeme>

Testing the Implementation

class TestRunner
{
    static void Main()
    {
        string sampleCode = @"
        var language = ""CustomLang"";
        print(language + "" is powerful!"");
        if (true) print(456);";

        LexicalProcessor processor = new LexicalProcessor(sampleCode);
        var resultTokens = processor.ProcessTokens();

        foreach (var token in resultTokens)
        {
            Console.WriteLine(token);
        }
    }
}