akbasic/tests/scanner_tokens.c

/**
 * @file scanner_tokens.c
 * @brief Tests the line tokenizer.
 */

#include "harness.h"

static akbasic_Environment *env(void)
{
    return HARNESS_RUNTIME.environment;
}

/* Scan a line and assert the token type at a given index. */
static void expect_token(int index, akbasic_TokenType want, const char *lexeme)
{
    TEST_REQUIRE(index < env()->nexttoken,
		 "expected a token at index %d, only %d were produced",
		 index, env()->nexttoken);
    if ( index >= env()->nexttoken ) {
	return;
    }
    TEST_REQUIRE_INT(env()->tokens[index].tokentype, want);
    if ( lexeme != NULL ) {
	TEST_REQUIRE_STR(env()->tokens[index].lexeme, lexeme);
    }
}

int main(void)
{
    char rewritten[AKBASIC_MAX_LINE_LENGTH];

    TEST_REQUIRE_OK(harness_start(NULL));

    /* Single-character operators. */
    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "+-*/^(),", NULL, 0));
    TEST_REQUIRE_INT(env()->nexttoken, 8);
    expect_token(0, AKBASIC_TOK_PLUS, "+");
    expect_token(1, AKBASIC_TOK_MINUS, "-");
    expect_token(2, AKBASIC_TOK_STAR, "*");
    expect_token(3, AKBASIC_TOK_LEFT_SLASH, "/");
    expect_token(4, AKBASIC_TOK_CARAT, "^");
    expect_token(5, AKBASIC_TOK_LEFT_PAREN, "(");
    expect_token(6, AKBASIC_TOK_RIGHT_PAREN, ")");
    expect_token(7, AKBASIC_TOK_COMMA, ",");

    /* Two-character comparisons, and the one-character fallbacks. */
    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "<= <> >= < > == = ", NULL, 0));
    TEST_REQUIRE_INT(env()->nexttoken, 7);
    expect_token(0, AKBASIC_TOK_LESS_THAN_EQUAL, NULL);
    expect_token(1, AKBASIC_TOK_NOT_EQUAL, NULL);
    expect_token(2, AKBASIC_TOK_GREATER_THAN_EQUAL, NULL);
    expect_token(3, AKBASIC_TOK_LESS_THAN, NULL);
    expect_token(4, AKBASIC_TOK_GREATER_THAN, NULL);
    expect_token(5, AKBASIC_TOK_EQUAL, NULL);
    expect_token(6, AKBASIC_TOK_ASSIGNMENT, NULL);

    /*
     * An operator in the final column survives, with no trailing space to help
     * it. matchNextChar cannot peek past the end of the line, and the reference
     * returns from that without setting a token type (basicscanner.go:272), so
     * `A# =` produced one token and the parse error that followed pointed
     * somewhere else. TODO.md section 6 item 14; this is its regression test.
     */
    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "A# =", NULL, 0));
    TEST_REQUIRE_INT(env()->nexttoken, 2);
    expect_token(0, AKBASIC_TOK_IDENTIFIER_INT, "A#");
    expect_token(1, AKBASIC_TOK_ASSIGNMENT, "=");

    /* The same for the other two, each alone in the final column. */
    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "A# <", NULL, 0));
    TEST_REQUIRE_INT(env()->nexttoken, 2);
    expect_token(1, AKBASIC_TOK_LESS_THAN, "<");
    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "A# >", NULL, 0));
    TEST_REQUIRE_INT(env()->nexttoken, 2);
    expect_token(1, AKBASIC_TOK_GREATER_THAN, ">");

    /* Literals. */
    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "PRINT 1.5", NULL, 0));
    expect_token(0, AKBASIC_TOK_COMMAND, "PRINT");
    expect_token(1, AKBASIC_TOK_LITERAL_FLOAT, "1.5");

    /*
     * A hex literal survives the scanner whole. The reference lets 'x' through
     * but stops at the hex digits after it (basicscanner.go:318), so "0xff"
     * lexed as "0x" plus an identifier "ff", the parser's base-16 branch was
     * unreachable and the README's hex support did not exist. TODO.md section 6
     * item 15; this is its regression test.
     */
    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "PRINT 0xff", NULL, 0));
    TEST_REQUIRE_INT(env()->nexttoken, 2);
    expect_token(1, AKBASIC_TOK_LITERAL_INT, "0xff");

    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "PRINT 0xDEAD", NULL, 0));
    expect_token(1, AKBASIC_TOK_LITERAL_INT, "0xDEAD");

    /* And the hex run ends where it should, rather than eating what follows. */
    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "PRINT 0x10 + 1", NULL, 0));
    TEST_REQUIRE_INT(env()->nexttoken, 4);
    expect_token(1, AKBASIC_TOK_LITERAL_INT, "0x10");
    expect_token(2, AKBASIC_TOK_PLUS, "+");
    expect_token(3, AKBASIC_TOK_LITERAL_INT, "1");

    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "PRINT \"HELLO\"", NULL, 0));
    expect_token(1, AKBASIC_TOK_LITERAL_STRING, "HELLO");

    /* An empty string literal really is empty, not a stray quote. */
    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "PRINT \"\"", NULL, 0));
    expect_token(1, AKBASIC_TOK_LITERAL_STRING, "");

    /* Identifier suffixes pick the token type. */
    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "A# B% C$ D", NULL, 0));
    expect_token(0, AKBASIC_TOK_IDENTIFIER_INT, "A#");
    expect_token(1, AKBASIC_TOK_IDENTIFIER_FLOAT, "B%");
    expect_token(2, AKBASIC_TOK_IDENTIFIER_STRING, "C$");
    expect_token(3, AKBASIC_TOK_IDENTIFIER, "D");

    /* Verbs are case-insensitive; a variable is not a verb. */
    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "print goto", NULL, 0));
    expect_token(0, AKBASIC_TOK_COMMAND, "print");
    expect_token(1, AKBASIC_TOK_COMMAND, "goto");

    /* Reserved words get their own token types, not COMMAND. */
    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "A# AND B# OR NOT C#", NULL, 0));
    expect_token(1, AKBASIC_TOK_AND, NULL);
    expect_token(3, AKBASIC_TOK_OR, NULL);
    expect_token(4, AKBASIC_TOK_NOT, NULL);

    /*
     * The line-number rule: an integer in token position 0 is consumed, not
     * emitted, and the line is rewritten to what follows it with leading spaces
     * stripped. The REPL stores the rewritten line as the program text, so this
     * is load-bearing rather than cosmetic.
     */
    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "10 PRINT 1", rewritten, sizeof(rewritten)));
    TEST_REQUIRE_STR(rewritten, "PRINT 1");
    TEST_REQUIRE_INT(env()->lineno, 10);
    expect_token(0, AKBASIC_TOK_COMMAND, "PRINT");
    expect_token(1, AKBASIC_TOK_LITERAL_INT, "1");

    /* An integer that is *not* in position 0 stays a literal. */
    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "PRINT 10", NULL, 0));
    expect_token(1, AKBASIC_TOK_LITERAL_INT, "10");

    /* REM ends the line: nothing after it becomes a token. */
    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "REM PRINT \"NOT SEEN\"", NULL, 0));
    TEST_REQUIRE_INT(env()->nexttoken, 0);

    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "10 REM A COMMENT", NULL, 0));
    TEST_REQUIRE_INT(env()->nexttoken, 0);
    TEST_REQUIRE_INT(env()->lineno, 10);

    /*
     * The "Reserved word in variable name" check never fires: the lexeme still
     * carries its suffix when the verb table is searched, so "PRINT$" misses and
     * becomes an ordinary string variable. Pinned as it stands -- TODO.md
     * section 12 item 16.
     */
    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "PRINT$ = 1", NULL, 0));
    TEST_REQUIRE(!HARNESS_RUNTIME.hasError,
		 "PRINT$ is currently accepted as a variable name; see TODO.md 12.16");
    expect_token(0, AKBASIC_TOK_IDENTIFIER_STRING, "PRINT$");

    /* An unknown character is diagnosed. */
    HARNESS_RUNTIME.hasError = false;
    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "PRINT ~", NULL, 0));
    TEST_REQUIRE(HARNESS_RUNTIME.hasError, "an unknown token must be diagnosed");

    /* A malformed number is diagnosed rather than silently becoming zero. */
    HARNESS_RUNTIME.hasError = false;
    TEST_REQUIRE_OK(akbasic_scanner_scan(&HARNESS_RUNTIME, "1x2 PRINT 1", NULL, 0));
    TEST_REQUIRE(HARNESS_RUNTIME.hasError,
		 "a malformed line number must be diagnosed, not converted to 0");
    TEST_REQUIRE(strstr(HARNESS_OUTPUT, "INTEGER CONVERSION ON '1x2'") != NULL,
		 "expected the reference's conversion message, got: %s", HARNESS_OUTPUT);

    TEST_REQUIRE_STATUS(akbasic_scanner_scan(&HARNESS_RUNTIME, NULL, NULL, 0), AKERR_NULLPOINTER);

    harness_stop();
    return akbasic_test_failures;
}