diff --git a/src/tui/browser/Viewer.tsx b/src/tui/browser/Viewer.tsx index 657d121..91c837c 100644 --- a/src/tui/browser/Viewer.tsx +++ b/src/tui/browser/Viewer.tsx @@ -1,7 +1,7 @@ import React from 'react'; import { Box, Text } from 'ink'; import { POU } from '../shared/types.js'; -import { tokenize, TokenKind } from './highlight.js'; +import { Token, tokenizeLines, TokenKind } from './highlight.js'; const COLORS: Record = { keyword: 'cyan', @@ -11,8 +11,7 @@ const COLORS: Record = { text: undefined, }; -function HighlightedLine({ line }: { line: string }): React.ReactElement { - const tokens = React.useMemo(() => tokenize(line), [line]); +function HighlightedTokens({ tokens }: { tokens: Token[] }): React.ReactElement { return ( {tokens.map((t, i) => ( @@ -40,16 +39,19 @@ export function Viewer({ pou, text, scrollTop, visibleRows }: ViewerProps): Reac ); } const lines = text.split(/\r?\n/); - const slice = lines.slice(scrollTop, scrollTop + visibleRows); + // Tokenize from line 0 so multi-line (* ... *) state is correct, then + // slice the visible window. Cheap; line count is bounded by file size. + const allTokens = React.useMemo(() => tokenizeLines(lines), [text]); + const sliceTokens = allTokens.slice(scrollTop, scrollTop + visibleRows); return ( {pou.name}.st ({pou.kind}, {pou.loc} L) - {slice.map((l, i) => ( + {sliceTokens.map((tokens, i) => ( {String(scrollTop + i + 1).padStart(4, ' ')} - + ))} diff --git a/src/tui/browser/highlight.ts b/src/tui/browser/highlight.ts index 61dccb6..8333e76 100644 --- a/src/tui/browser/highlight.ts +++ b/src/tui/browser/highlight.ts @@ -42,17 +42,25 @@ const TYPES = new Set([ const IDENT_RE = /[A-Z_][A-Z0-9_]*/; -export function tokenize(line: string): Token[] { +/** + * Tokenize a single line. If `openComment` is true, the line is treated as + * starting inside a (* ... *) block; output includes a `commentLeftOpen` + * flag so callers can continue the state across lines. + */ +export interface TokenizeResult { + tokens: Token[]; + /** True if the line ended without closing a (* block. */ + commentLeftOpen: boolean; +} + +export function tokenizeWithState(line: string, openComment: boolean): TokenizeResult { const out: Token[] = []; let i = 0; + let inComment = openComment; let pending = ''; const flushPending = () => { if (!pending) return; - // Walk pending and split at every uppercase identifier boundary, so - // identifiers (whether keyword/type or plain) come out as their own - // tokens. Anything else (whitespace, punctuation, lowercase ids) is - // emitted as 'text'. const re = /[A-Z_][A-Z0-9_]*/g; let last = 0; let m: RegExpExecArray | null; @@ -68,18 +76,29 @@ export function tokenize(line: string): Token[] { pending = ''; }; + // If line started inside a comment, eat up to "*)" or end-of-line. + if (inComment) { + const end = line.indexOf('*)'); + if (end < 0) { + out.push({ kind: 'comment', text: line }); + return { tokens: out, commentLeftOpen: true }; + } + out.push({ kind: 'comment', text: line.slice(0, end + 2) }); + i = end + 2; + inComment = false; + } + while (i < line.length) { - // (* ... *) inline comment + // (* ... *) — may be inline or open-ended. if (line[i] === '(' && line[i + 1] === '*') { flushPending(); const end = line.indexOf('*)', i + 2); if (end < 0) { out.push({ kind: 'comment', text: line.slice(i) }); - i = line.length; - } else { - out.push({ kind: 'comment', text: line.slice(i, end + 2) }); - i = end + 2; + return { tokens: out, commentLeftOpen: true }; } + out.push({ kind: 'comment', text: line.slice(i, end + 2) }); + i = end + 2; continue; } // // line comment @@ -89,7 +108,7 @@ export function tokenize(line: string): Token[] { i = line.length; continue; } - // 'single' or "double" strings (no escape handling beyond doubled quotes) + // 'single' or "double" strings if (line[i] === "'" || line[i] === '"') { flushPending(); const quote = line[i]; @@ -104,6 +123,21 @@ export function tokenize(line: string): Token[] { i++; } flushPending(); - // Normalize: empty trailing 'text' tokens are fine; we keep them so the line's column structure is preserved. + return { tokens: out, commentLeftOpen: false }; +} + +export function tokenize(line: string): Token[] { + return tokenizeWithState(line, false).tokens; +} + +/** Tokenize a contiguous block of lines, threading (* ... *) state across line boundaries. */ +export function tokenizeLines(lines: string[]): Token[][] { + const out: Token[][] = []; + let openComment = false; + for (const line of lines) { + const r = tokenizeWithState(line, openComment); + out.push(r.tokens); + openComment = r.commentLeftOpen; + } return out; } diff --git a/tests/tui/highlight.test.ts b/tests/tui/highlight.test.ts index b0f7b73..ad6873d 100644 --- a/tests/tui/highlight.test.ts +++ b/tests/tui/highlight.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { tokenize, TokenKind } from '../../src/tui/browser/highlight'; +import { tokenize, tokenizeLines, TokenKind } from '../../src/tui/browser/highlight'; function kinds(line: string): TokenKind[] { return tokenize(line).map((t) => t.kind); @@ -55,3 +55,36 @@ describe('tokenize', () => { expect(texts('s := "hi";')).toContain('"hi"'); }); }); + +describe('tokenizeLines', () => { + it('treats every line of a multi-line (* ... *) block as comment', () => { + const lines = [ + 'x := 1; (* start', + ' middle keep PROGRAM unhighlighted', + ' still in comment', + 'end *) y := 2;', + ]; + const out = tokenizeLines(lines); + // line 0: leading "x := 1; " is text/keyword-free, then "(* start" is comment + expect(out[0].some((t) => t.kind === 'comment' && t.text.includes('(* start'))).toBe(true); + // line 1: ENTIRE line should be a single comment token + expect(out[1]).toEqual([{ kind: 'comment', text: ' middle keep PROGRAM unhighlighted' }]); + // line 2: ENTIRE line is comment + expect(out[2]).toEqual([{ kind: 'comment', text: ' still in comment' }]); + // line 3: starts comment, then `*) y := 2;` is text after close + expect(out[3].some((t) => t.kind === 'comment' && t.text.startsWith('end *)'))).toBe(true); + expect(out[3].some((t) => t.kind === 'text' && t.text.includes('y := 2'))).toBe(true); + }); + + it('handles single-line input identically to tokenize', () => { + const single = 'PROGRAM PLC_PRG'; + expect(tokenizeLines([single])).toEqual([tokenize(single)]); + }); + + it('keeps PROGRAM keyword highlighted on a line with no open block', () => { + const lines = ['PROGRAM A', '(* block *) END_PROGRAM']; + const out = tokenizeLines(lines); + expect(out[0].find((t) => t.text === 'PROGRAM')?.kind).toBe('keyword'); + expect(out[1].find((t) => t.text === 'END_PROGRAM')?.kind).toBe('keyword'); + }); +});