tui(viewer): join multi-line (* ... *) comments across line boundaries

Adds tokenizeWithState(line, openComment) -> {tokens, commentLeftOpen}
and tokenizeLines(allLines) which threads the open-comment flag across
lines.

Viewer now tokenizes from line 0 (not just the visible slice) so the
state going into the visible window is correct, then renders the
slice. A line wholly inside a (* block is emitted as a single
'comment' token; PROGRAM and other keywords on those lines no longer
get falsely highlighted.

tokenize(line) kept as a thin wrapper for the single-line callers
(tests, future use).

src/tui/browser/Viewer.tsx

@@ -1,7 +1,7 @@
 import React from 'react';
 import { Box, Text } from 'ink';
 import { POU } from '../shared/types.js';
-import { tokenize, TokenKind } from './highlight.js';
+import { Token, tokenizeLines, TokenKind } from './highlight.js';
 
 const COLORS: Record<TokenKind, string | undefined> = {
   keyword: 'cyan',
@@ -11,8 +11,7 @@ const COLORS: Record<TokenKind, string | undefined> = {
   text: undefined,
 };
 
-function HighlightedLine({ line }: { line: string }): React.ReactElement {
-  const tokens = React.useMemo(() => tokenize(line), [line]);
+function HighlightedTokens({ tokens }: { tokens: Token[] }): React.ReactElement {
   return (
     <Text>
       {tokens.map((t, i) => (
@@ -40,16 +39,19 @@ export function Viewer({ pou, text, scrollTop, visibleRows }: ViewerProps): Reac
     );
   }
   const lines = text.split(/\r?\n/);
-  const slice = lines.slice(scrollTop, scrollTop + visibleRows);
+  // Tokenize from line 0 so multi-line (* ... *) state is correct, then
+  // slice the visible window. Cheap; line count is bounded by file size.
+  const allTokens = React.useMemo(() => tokenizeLines(lines), [text]);
+  const sliceTokens = allTokens.slice(scrollTop, scrollTop + visibleRows);
   return (
     <Box flexDirection="column">
       <Text bold>
         {pou.name}.st  ({pou.kind}, {pou.loc} L)
       </Text>
-      {slice.map((l, i) => (
+      {sliceTokens.map((tokens, i) => (
         <Text key={scrollTop + i}>
           <Text dimColor>{String(scrollTop + i + 1).padStart(4, ' ')}  </Text>
-          <HighlightedLine line={l} />
+          <HighlightedTokens tokens={tokens} />
         </Text>
       ))}
     </Box>

src/tui/browser/highlight.ts

@@ -42,17 +42,25 @@ const TYPES = new Set([
 
 const IDENT_RE = /[A-Z_][A-Z0-9_]*/;
 
-export function tokenize(line: string): Token[] {
+/**
+ * Tokenize a single line. If `openComment` is true, the line is treated as
+ * starting inside a (* ... *) block; output includes a `commentLeftOpen`
+ * flag so callers can continue the state across lines.
+ */
+export interface TokenizeResult {
+  tokens: Token[];
+  /** True if the line ended without closing a (* block. */
+  commentLeftOpen: boolean;
+}
+
+export function tokenizeWithState(line: string, openComment: boolean): TokenizeResult {
   const out: Token[] = [];
   let i = 0;
+  let inComment = openComment;
   let pending = '';
 
   const flushPending = () => {
     if (!pending) return;
-    // Walk pending and split at every uppercase identifier boundary, so
-    // identifiers (whether keyword/type or plain) come out as their own
-    // tokens. Anything else (whitespace, punctuation, lowercase ids) is
-    // emitted as 'text'.
     const re = /[A-Z_][A-Z0-9_]*/g;
     let last = 0;
     let m: RegExpExecArray | null;
@@ -68,18 +76,29 @@ export function tokenize(line: string): Token[] {
     pending = '';
   };
 
+  // If line started inside a comment, eat up to "*)" or end-of-line.
+  if (inComment) {
+    const end = line.indexOf('*)');
+    if (end < 0) {
+      out.push({ kind: 'comment', text: line });
+      return { tokens: out, commentLeftOpen: true };
+    }
+    out.push({ kind: 'comment', text: line.slice(0, end + 2) });
+    i = end + 2;
+    inComment = false;
+  }
+
   while (i < line.length) {
-    // (* ... *) inline comment
+    // (* ... *) — may be inline or open-ended.
     if (line[i] === '(' && line[i + 1] === '*') {
       flushPending();
       const end = line.indexOf('*)', i + 2);
       if (end < 0) {
         out.push({ kind: 'comment', text: line.slice(i) });
-        i = line.length;
-      } else {
-        out.push({ kind: 'comment', text: line.slice(i, end + 2) });
-        i = end + 2;
+        return { tokens: out, commentLeftOpen: true };
       }
+      out.push({ kind: 'comment', text: line.slice(i, end + 2) });
+      i = end + 2;
       continue;
     }
     // // line comment
@@ -89,7 +108,7 @@ export function tokenize(line: string): Token[] {
       i = line.length;
       continue;
     }
-    // 'single' or "double" strings (no escape handling beyond doubled quotes)
+    // 'single' or "double" strings
     if (line[i] === "'" || line[i] === '"') {
       flushPending();
       const quote = line[i];
@@ -104,6 +123,21 @@ export function tokenize(line: string): Token[] {
     i++;
   }
   flushPending();
-  // Normalize: empty trailing 'text' tokens are fine; we keep them so the line's column structure is preserved.
+  return { tokens: out, commentLeftOpen: false };
+}
+
+export function tokenize(line: string): Token[] {
+  return tokenizeWithState(line, false).tokens;
+}
+
+/** Tokenize a contiguous block of lines, threading (* ... *) state across line boundaries. */
+export function tokenizeLines(lines: string[]): Token[][] {
+  const out: Token[][] = [];
+  let openComment = false;
+  for (const line of lines) {
+    const r = tokenizeWithState(line, openComment);
+    out.push(r.tokens);
+    openComment = r.commentLeftOpen;
+  }
   return out;
 }

tests/tui/highlight.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from 'vitest';
-import { tokenize, TokenKind } from '../../src/tui/browser/highlight';
+import { tokenize, tokenizeLines, TokenKind } from '../../src/tui/browser/highlight';
 
 function kinds(line: string): TokenKind[] {
   return tokenize(line).map((t) => t.kind);
@@ -55,3 +55,36 @@ describe('tokenize', () => {
     expect(texts('s := "hi";')).toContain('"hi"');
   });
 });
+
+describe('tokenizeLines', () => {
+  it('treats every line of a multi-line (* ... *) block as comment', () => {
+    const lines = [
+      'x := 1; (* start',
+      '  middle keep PROGRAM unhighlighted',
+      '  still in comment',
+      'end *) y := 2;',
+    ];
+    const out = tokenizeLines(lines);
+    // line 0: leading "x := 1; " is text/keyword-free, then "(* start" is comment
+    expect(out[0].some((t) => t.kind === 'comment' && t.text.includes('(* start'))).toBe(true);
+    // line 1: ENTIRE line should be a single comment token
+    expect(out[1]).toEqual([{ kind: 'comment', text: '  middle keep PROGRAM unhighlighted' }]);
+    // line 2: ENTIRE line is comment
+    expect(out[2]).toEqual([{ kind: 'comment', text: '  still in comment' }]);
+    // line 3: starts comment, then `*) y := 2;` is text after close
+    expect(out[3].some((t) => t.kind === 'comment' && t.text.startsWith('end *)'))).toBe(true);
+    expect(out[3].some((t) => t.kind === 'text' && t.text.includes('y := 2'))).toBe(true);
+  });
+
+  it('handles single-line input identically to tokenize', () => {
+    const single = 'PROGRAM PLC_PRG';
+    expect(tokenizeLines([single])).toEqual([tokenize(single)]);
+  });
+
+  it('keeps PROGRAM keyword highlighted on a line with no open block', () => {
+    const lines = ['PROGRAM A', '(* block *) END_PROGRAM'];
+    const out = tokenizeLines(lines);
+    expect(out[0].find((t) => t.text === 'PROGRAM')?.kind).toBe('keyword');
+    expect(out[1].find((t) => t.text === 'END_PROGRAM')?.kind).toBe('keyword');
+  });
+});