Andrew Kesterson
583c0abbd2
The Go reference is deprecated and will not be updated, so the two projects are no longer required to match. Recorded as TODO.md section 0.1, first thing in the file, because it silently reverses the premise several later sections were written on -- an agent reading section 6 without it will park work that is no longer blocked. Section 6 item 16 was the item waiting on exactly this and is now fixed: the keyword tables are searched on the base name with any type suffix stripped, so PRINT$, LEN# and GOTO% are refused as variable names and the reference's own diagnostic stops being dead code. It cost the one golden case predicted, and the cost was nothing -- renaming a variable in strreverse.bas left its expected output byte-for-byte unchanged. AKBASIC_KNOWN_FAILING_TESTS is empty as a result and known_reference_defects.c is gone. Section 6 items 1-9 and 11 are reopened as an ordinary defect list. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
198 lines
8.8 KiB
C
198 lines
8.8 KiB
C
/**
|
|
* @file scanner_tokens.c
|
|
* @brief Tests the line tokenizer.
|
|
*/
|
|
|
|
#include "harness.h"
|
|
|
|
static akbasic_Environment *env(void)
|
|
{
|
|
return HARNESS_RUNTIME.environment;
|
|
}
|
|
|
|
/* Scan a line and assert the token type at a given index. */
|
|
static void expect_token(int index, akbasic_TokenType want, const char *lexeme)
|
|
{
|
|
TEST_REQUIRE(index < env()->nexttoken,
|
|
"expected a token at index %d, only %d were produced",
|
|
index, env()->nexttoken);
|
|
if ( index >= env()->nexttoken ) {
|
|
return;
|
|
}
|
|
TEST_REQUIRE_INT(env()->tokens[index].tokentype, want);
|
|
if ( lexeme != NULL ) {
|
|
TEST_REQUIRE_STR(env()->tokens[index].lexeme, lexeme);
|
|
}
|
|
}
|
|
|
|
int main(void)
|
|
{
|
|
char rewritten[AKBASIC_MAX_LINE_LENGTH];
|
|
|
|
TEST_REQUIRE_OK(harness_start(NULL));
|
|
|
|
/* Single-character operators. */
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "+-*/^(),", NULL, 0));
|
|
TEST_REQUIRE_INT(env()->nexttoken, 8);
|
|
expect_token(0, AKBASIC_TOK_PLUS, "+");
|
|
expect_token(1, AKBASIC_TOK_MINUS, "-");
|
|
expect_token(2, AKBASIC_TOK_STAR, "*");
|
|
expect_token(3, AKBASIC_TOK_LEFT_SLASH, "/");
|
|
expect_token(4, AKBASIC_TOK_CARAT, "^");
|
|
expect_token(5, AKBASIC_TOK_LEFT_PAREN, "(");
|
|
expect_token(6, AKBASIC_TOK_RIGHT_PAREN, ")");
|
|
expect_token(7, AKBASIC_TOK_COMMA, ",");
|
|
|
|
/* Two-character comparisons, and the one-character fallbacks. */
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "<= <> >= < > == = ", NULL, 0));
|
|
TEST_REQUIRE_INT(env()->nexttoken, 7);
|
|
expect_token(0, AKBASIC_TOK_LESS_THAN_EQUAL, NULL);
|
|
expect_token(1, AKBASIC_TOK_NOT_EQUAL, NULL);
|
|
expect_token(2, AKBASIC_TOK_GREATER_THAN_EQUAL, NULL);
|
|
expect_token(3, AKBASIC_TOK_LESS_THAN, NULL);
|
|
expect_token(4, AKBASIC_TOK_GREATER_THAN, NULL);
|
|
expect_token(5, AKBASIC_TOK_EQUAL, NULL);
|
|
expect_token(6, AKBASIC_TOK_ASSIGNMENT, NULL);
|
|
|
|
/*
|
|
* An operator in the final column survives, with no trailing space to help
|
|
* it. matchNextChar cannot peek past the end of the line, and the reference
|
|
* returns from that without setting a token type (basicscanner.go:272), so
|
|
* `A# =` produced one token and the parse error that followed pointed
|
|
* somewhere else. TODO.md section 6 item 14; this is its regression test.
|
|
*/
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "A# =", NULL, 0));
|
|
TEST_REQUIRE_INT(env()->nexttoken, 2);
|
|
expect_token(0, AKBASIC_TOK_IDENTIFIER_INT, "A#");
|
|
expect_token(1, AKBASIC_TOK_ASSIGNMENT, "=");
|
|
|
|
/* The same for the other two, each alone in the final column. */
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "A# <", NULL, 0));
|
|
TEST_REQUIRE_INT(env()->nexttoken, 2);
|
|
expect_token(1, AKBASIC_TOK_LESS_THAN, "<");
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "A# >", NULL, 0));
|
|
TEST_REQUIRE_INT(env()->nexttoken, 2);
|
|
expect_token(1, AKBASIC_TOK_GREATER_THAN, ">");
|
|
|
|
/* Literals. */
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "PRINT 1.5", NULL, 0));
|
|
expect_token(0, AKBASIC_TOK_COMMAND, "PRINT");
|
|
expect_token(1, AKBASIC_TOK_LITERAL_FLOAT, "1.5");
|
|
|
|
/*
|
|
* A hex literal survives the scanner whole. The reference lets 'x' through
|
|
* but stops at the hex digits after it (basicscanner.go:318), so "0xff"
|
|
* lexed as "0x" plus an identifier "ff", the parser's base-16 branch was
|
|
* unreachable and the README's hex support did not exist. TODO.md section 6
|
|
* item 15; this is its regression test.
|
|
*/
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "PRINT 0xff", NULL, 0));
|
|
TEST_REQUIRE_INT(env()->nexttoken, 2);
|
|
expect_token(1, AKBASIC_TOK_LITERAL_INT, "0xff");
|
|
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "PRINT 0xDEAD", NULL, 0));
|
|
expect_token(1, AKBASIC_TOK_LITERAL_INT, "0xDEAD");
|
|
|
|
/* And the hex run ends where it should, rather than eating what follows. */
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "PRINT 0x10 + 1", NULL, 0));
|
|
TEST_REQUIRE_INT(env()->nexttoken, 4);
|
|
expect_token(1, AKBASIC_TOK_LITERAL_INT, "0x10");
|
|
expect_token(2, AKBASIC_TOK_PLUS, "+");
|
|
expect_token(3, AKBASIC_TOK_LITERAL_INT, "1");
|
|
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "PRINT \"HELLO\"", NULL, 0));
|
|
expect_token(1, AKBASIC_TOK_LITERAL_STRING, "HELLO");
|
|
|
|
/* An empty string literal really is empty, not a stray quote. */
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "PRINT \"\"", NULL, 0));
|
|
expect_token(1, AKBASIC_TOK_LITERAL_STRING, "");
|
|
|
|
/* Identifier suffixes pick the token type. */
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "A# B% C$ D", NULL, 0));
|
|
expect_token(0, AKBASIC_TOK_IDENTIFIER_INT, "A#");
|
|
expect_token(1, AKBASIC_TOK_IDENTIFIER_FLOAT, "B%");
|
|
expect_token(2, AKBASIC_TOK_IDENTIFIER_STRING, "C$");
|
|
expect_token(3, AKBASIC_TOK_IDENTIFIER, "D");
|
|
|
|
/* Verbs are case-insensitive; a variable is not a verb. */
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "print goto", NULL, 0));
|
|
expect_token(0, AKBASIC_TOK_COMMAND, "print");
|
|
expect_token(1, AKBASIC_TOK_COMMAND, "goto");
|
|
|
|
/* Reserved words get their own token types, not COMMAND. */
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "A# AND B# OR NOT C#", NULL, 0));
|
|
expect_token(1, AKBASIC_TOK_AND, NULL);
|
|
expect_token(3, AKBASIC_TOK_OR, NULL);
|
|
expect_token(4, AKBASIC_TOK_NOT, NULL);
|
|
|
|
/*
|
|
* The line-number rule: an integer in token position 0 is consumed, not
|
|
* emitted, and the line is rewritten to what follows it with leading spaces
|
|
* stripped. The REPL stores the rewritten line as the program text, so this
|
|
* is load-bearing rather than cosmetic.
|
|
*/
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "10 PRINT 1", rewritten, sizeof(rewritten)));
|
|
TEST_REQUIRE_STR(rewritten, "PRINT 1");
|
|
TEST_REQUIRE_INT(env()->lineno, 10);
|
|
expect_token(0, AKBASIC_TOK_COMMAND, "PRINT");
|
|
expect_token(1, AKBASIC_TOK_LITERAL_INT, "1");
|
|
|
|
/* An integer that is *not* in position 0 stays a literal. */
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "PRINT 10", NULL, 0));
|
|
expect_token(1, AKBASIC_TOK_LITERAL_INT, "10");
|
|
|
|
/* REM ends the line: nothing after it becomes a token. */
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "REM PRINT \"NOT SEEN\"", NULL, 0));
|
|
TEST_REQUIRE_INT(env()->nexttoken, 0);
|
|
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "10 REM A COMMENT", NULL, 0));
|
|
TEST_REQUIRE_INT(env()->nexttoken, 0);
|
|
TEST_REQUIRE_INT(env()->lineno, 10);
|
|
|
|
/*
|
|
* A verb name with a type suffix stuck on it is refused as a variable name.
|
|
* The reference searches the keyword tables with the suffix still attached
|
|
* (basicscanner.go:349), so "PRINT$" misses every table, its own "Reserved
|
|
* word in variable name" diagnostic is dead code, and `PRINT$ = 1` is
|
|
* quietly accepted. TODO.md section 6 item 16; this is its regression test.
|
|
*/
|
|
HARNESS_RUNTIME.hasError = false;
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "PRINT$ = 1", NULL, 0));
|
|
TEST_REQUIRE(HARNESS_RUNTIME.hasError, "PRINT$ must be refused as a variable name");
|
|
|
|
/* The same for a function name, and for the other two suffixes. */
|
|
HARNESS_RUNTIME.hasError = false;
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "LEN# = 1", NULL, 0));
|
|
TEST_REQUIRE(HARNESS_RUNTIME.hasError, "LEN# must be refused as a variable name");
|
|
HARNESS_RUNTIME.hasError = false;
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "GOTO% = 1", NULL, 0));
|
|
TEST_REQUIRE(HARNESS_RUNTIME.hasError, "GOTO%% must be refused as a variable name");
|
|
|
|
/*
|
|
* And an ordinary name that merely *contains* a verb is still fine, which is
|
|
* what says the check compares the whole base name rather than a prefix.
|
|
*/
|
|
HARNESS_RUNTIME.hasError = false;
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "PRINTER$ = 1", NULL, 0));
|
|
TEST_REQUIRE(!HARNESS_RUNTIME.hasError, "PRINTER$ is not a reserved word");
|
|
expect_token(0, AKBASIC_TOK_IDENTIFIER_STRING, "PRINTER$");
|
|
|
|
/* An unknown character is diagnosed. */
|
|
HARNESS_RUNTIME.hasError = false;
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "PRINT ~", NULL, 0));
|
|
TEST_REQUIRE(HARNESS_RUNTIME.hasError, "an unknown token must be diagnosed");
|
|
|
|
/* A malformed number is diagnosed rather than silently becoming zero. */
|
|
HARNESS_RUNTIME.hasError = false;
|
|
TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "1x2 PRINT 1", NULL, 0));
|
|
TEST_REQUIRE(HARNESS_RUNTIME.hasError,
|
|
"a malformed line number must be diagnosed, not converted to 0");
|
|
TEST_REQUIRE(strstr(HARNESS_OUTPUT, "INTEGER CONVERSION ON '1x2'") != NULL,
|
|
"expected the reference's conversion message, got: %s", HARNESS_OUTPUT);
|
|
|
|
TEST_REQUIRE_STATUS(akbasic_scanner_scan(&HARNESS_RUNTIME, NULL, NULL, 0), AKERR_NULLPOINTER);
|
|
|
|
harness_stop();
|
|
return akbasic_test_failures;
|
|
}
|