From dde1d91c6e2a7877d091803207b4ce47432823d4 Mon Sep 17 00:00:00 2001 From: Tachikoma Date: Tue, 4 Aug 2026 08:46:51 -0400 Subject: [PATCH] Reset the scanner's leftover token type between lines The REM early-exit leaves tokentype holding AKBASIC_TOK_REM, and the scan loop's post-switch check reads it before the next line's first character has assigned anything. A line opening with whitespace then re-triggered the REM break and scanned to nothing: every indented line after a REM was silently skipped. Numbered programs never saw it -- the line number is the first token and overwrites the leftover -- which is why the whole golden corpus missed it and the unnumbered, indented galaga.bas found it. Co-authored-by: andrew Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XiGgpHuXUm2mR4Wzndw3dc --- src/scanner.c | 11 +++++++++++ tests/language/statements/rem_indented_line.bas | 14 ++++++++++++++ tests/language/statements/rem_indented_line.txt | 3 +++ 3 files changed, 28 insertions(+) create mode 100644 tests/language/statements/rem_indented_line.bas create mode 100644 tests/language/statements/rem_indented_line.txt diff --git a/src/scanner.c b/src/scanner.c index 2f7ed40..c49fc69 100644 --- a/src/scanner.c +++ b/src/scanner.c @@ -23,6 +23,7 @@ akerr_ErrorContext *akbasic_scanner_zero(akbasic_Runtime *obj) obj->current = 0; obj->start = 0; obj->hasError = false; + obj->tokentype = AKBASIC_TOK_UNDEFINED; SUCCEED_RETURN(errctx); } @@ -409,6 +410,16 @@ akerr_ErrorContext *akbasic_scanner_scan(akbasic_Runtime *obj, const char *line, obj->current = 0; obj->start = 0; obj->hasError = false; + /* + * The `REM` early-exit below leaves `tokentype` holding AKBASIC_TOK_REM, + * and the loop's post-switch check reads it before the first character of + * the *next* line has assigned anything. A line whose first character + * carries no token of its own -- leading whitespace -- then re-triggered + * the REM break and scanned to nothing: every indented line after a REM + * was silently skipped. A numbered program never saw it, because the line + * number is the first token and overwrites the leftover. + */ + obj->tokentype = AKBASIC_TOK_UNDEFINED; /* * Cleared here rather than by each caller, so the flag always describes the * line this call just scanned. It used to be cleared only in diff --git a/tests/language/statements/rem_indented_line.bas b/tests/language/statements/rem_indented_line.bas new file mode 100644 index 0000000..5628d03 --- /dev/null +++ b/tests/language/statements/rem_indented_line.bas @@ -0,0 +1,14 @@ +REM The line after this comment is indented, and it must still run: the +REM scanner's REM early-exit used to leave TOK_REM armed, and the next +REM line's leading whitespace re-triggered it -- every indented line +REM after a REM was silently skipped. Unnumbered on purpose: a numbered +REM line's first token overwrites the leftover and hides the defect. +PRINT 1 +REM an indented statement follows + PRINT 2 +REM an indented multi-line DEF body, the shape that found it +DEF F(N#) + REM a comment inside the body + RETURN N# + 5 +PRINT F(10) +END diff --git a/tests/language/statements/rem_indented_line.txt b/tests/language/statements/rem_indented_line.txt new file mode 100644 index 0000000..4b5aa34 --- /dev/null +++ b/tests/language/statements/rem_indented_line.txt @@ -0,0 +1,3 @@ +1 +2 +15