akbasic/src/scanner.c

/**
 * @file scanner.c
 * @brief Implements the line tokenizer.
 */

#include <ctype.h>
#include <inttypes.h>
#include <stdio.h>
#include <string.h>

#include <akerror.h>
#include <akstdlib.h>

#include <akbasic/error.h>
#include <akbasic/scanner.h>
#include <akbasic/verbs.h>

akerr_ErrorContext *akbasic_scanner_zero(akbasic_Runtime *obj)
{
    PREPARE_ERROR(errctx);

    FAIL_ZERO_RETURN(errctx, (obj != NULL), AKERR_NULLPOINTER, "NULL runtime in scanner zero");
    obj->current = 0;
    obj->start = 0;
    obj->hasError = false;
    SUCCEED_RETURN(errctx);
}

static bool is_at_end(akbasic_Runtime *obj)
{
    return (obj->current >= (int)strlen(obj->line));
}

static bool peek(akbasic_Runtime *obj, char *dest)
{
    if ( is_at_end(obj) ) {
	return false;
    }
    *dest = obj->line[obj->current];
    return true;
}

static bool peek_next(akbasic_Runtime *obj, char *dest)
{
    if ( (obj->current + 1) >= (int)strlen(obj->line) ) {
	return false;
    }
    *dest = obj->line[obj->current + 1];
    return true;
}

/*
 * The lexeme is the span [start, current), with two special cases the reference
 * relies on: at end of line it runs to the end, and a zero-width span yields the
 * single character at `start` -- unless we are closing a string literal, where
 * an empty span really is the empty string.
 */
static akerr_ErrorContext *get_lexeme(akbasic_Runtime *obj, char *dest, size_t len)
{
    PREPARE_ERROR(errctx);
    int linelen = (int)strlen(obj->line);
    int span = 0;

    if ( obj->current == linelen ) {
	span = linelen - obj->start;
    } else if ( obj->start == obj->current ) {
	if ( obj->tokentype == AKBASIC_TOK_LITERAL_STRING ) {
	    dest[0] = '\0';
	    SUCCEED_RETURN(errctx);
	}
	span = 1;
    } else {
	span = obj->current - obj->start;
    }
    FAIL_ZERO_RETURN(errctx, (span >= 0 && (size_t)span < len), AKBASIC_ERR_BOUNDS,
		     "Lexeme of %d characters exceeds the %zu character limit", span, len - 1);
    memcpy(dest, obj->line + obj->start, (size_t)span);
    dest[span] = '\0';
    SUCCEED_RETURN(errctx);
}

static akerr_ErrorContext *add_token(akbasic_Runtime *obj, akbasic_TokenType token, const char *lexeme)
{
    PREPARE_ERROR(errctx);
    akbasic_Environment *env = obj->environment;

    FAIL_ZERO_RETURN(errctx, (env->nexttoken < AKBASIC_MAX_TOKENS), AKBASIC_ERR_BOUNDS,
		     "Line %" PRId64 " has more than %d tokens",
		     env->lineno, AKBASIC_MAX_TOKENS);
    FAIL_ZERO_RETURN(errctx, (strlen(lexeme) < AKBASIC_MAX_LINE_LENGTH), AKBASIC_ERR_BOUNDS,
		     "Token lexeme exceeds the %d character limit", AKBASIC_MAX_LINE_LENGTH - 1);
    env->tokens[env->nexttoken].tokentype = token;
    env->tokens[env->nexttoken].lineno = env->lineno;
    strncpy(env->tokens[env->nexttoken].lexeme, lexeme, AKBASIC_MAX_LINE_LENGTH - 1);
    env->tokens[env->nexttoken].lexeme[AKBASIC_MAX_LINE_LENGTH - 1] = '\0';
    env->nexttoken += 1;
    SUCCEED_RETURN(errctx);
}

/* Consume one more character when it matches, choosing between two token types. */
static bool match_next_char(akbasic_Runtime *obj, char cm, akbasic_TokenType truetype, akbasic_TokenType falsetype)
{
    char nc = '\0';

    if ( !peek(obj, &nc) ) {
	/*
	 * Nothing left to peek at, so the operator is whatever it is on its
	 * own. The reference returns here *without* setting a type
	 * (basicscanner.go:272), which silently drops a comparison operator in
	 * the final column of a line: `A# =` produced one token, not two, and
	 * the parse error that followed pointed somewhere else entirely.
	 * TODO.md section 6 item 14.
	 */
	obj->tokentype = falsetype;
	return false;
    }
    if ( nc == cm ) {
	obj->current += 1;
	obj->tokentype = truetype;
	return true;
    }
    obj->tokentype = falsetype;
    return false;
}

static akerr_ErrorContext *match_string(akbasic_Runtime *obj)
{
    PREPARE_ERROR(errctx);
    char c = '\0';

    while ( !is_at_end(obj) ) {
	if ( !peek(obj, &c) ) {
	    PASS(errctx, akbasic_runtime_error(obj, AKBASIC_ERRCLASS_PARSE,
					       "UNTERMINATED STRING LITERAL\n"));
	    obj->hasError = true;
	    SUCCEED_RETURN(errctx);
	}
	if ( c == '"' ) {
	    break;
	}
	obj->current += 1;
    }
    obj->tokentype = AKBASIC_TOK_LITERAL_STRING;
    SUCCEED_RETURN(errctx);
}

static akerr_ErrorContext *match_number(akbasic_Runtime *obj)
{
    PREPARE_ERROR(errctx);
    bool linenumber = (obj->environment->nexttoken == 0);
    char lexeme[AKBASIC_MAX_LINE_LENGTH];
    char c = '\0';
    char nc = '\0';
    int64_t lineno = 0;
    long long converted = 0;
    bool hex = false;

    obj->tokentype = AKBASIC_TOK_LITERAL_INT;
    while ( !is_at_end(obj) ) {
	(void)peek(obj, &c);
	if ( c == '.' ) {
	    if ( !peek_next(obj, &nc) || !isdigit((unsigned char)nc) ) {
		PASS(errctx, akbasic_runtime_error(obj, AKBASIC_ERRCLASS_PARSE,
						   "INVALID FLOATING POINT LITERAL\n"));
		obj->hasError = true;
		SUCCEED_RETURN(errctx);
	    }
	    obj->tokentype = AKBASIC_TOK_LITERAL_FLOAT;
	} else if ( c == 'x' && !hex ) {
	    /*
	     * An 'x' in a number run. The reference lets it through and then
	     * stops at the first hex digit after it (basicscanner.go:318), so
	     * `0xff` lexed as `0x` followed by an identifier `ff`, the parser's
	     * base-16 branch was unreachable, and the README's hex support did
	     * not exist. TODO.md section 6 item 15. Once the 'x' is seen, keep
	     * going over hex digits.
	     *
	     * Accepted anywhere in the run rather than only after a leading `0`,
	     * which is the reference's rule and is worth keeping: it is what
	     * makes `1x2` one malformed token that the line-number conversion
	     * then diagnoses by name, instead of two tokens that fail somewhere
	     * less helpful.
	     */
	    hex = true;
	} else if ( hex ) {
	    if ( !isxdigit((unsigned char)c) ) {
		break;
	    }
	} else if ( !isdigit((unsigned char)c) ) {
	    break;
	}
	obj->current += 1;
    }

    if ( obj->tokentype == AKBASIC_TOK_LITERAL_INT && linenumber ) {
	PASS(errctx, get_lexeme(obj, lexeme, sizeof(lexeme)));
	ATTEMPT {
	    CATCH(errctx, aksl_atoll(lexeme, &converted));
	    lineno = (int64_t)converted;
	} CLEANUP {
	} PROCESS(errctx) {
	} HANDLE_DEFAULT(errctx) {
	    char message[AKBASIC_MAX_LINE_LENGTH + 32];
	    snprintf(message, sizeof(message), "INTEGER CONVERSION ON '%s'", lexeme);
	    /*
	     * Reporting can itself fail if the sink is broken. Nothing useful
	     * remains to be done about that here, so record the flag and let the
	     * next operation surface it.
	     */
	    IGNORE(akbasic_runtime_error(obj, AKBASIC_ERRCLASS_PARSE, message));
	    obj->hasError = true;
	} FINISH(errctx, false);

	if ( obj->hasError ) {
	    SUCCEED_RETURN(errctx);
	}
	obj->environment->lineno = lineno;
	obj->tokentype = AKBASIC_TOK_LINE_NUMBER;
    }
    SUCCEED_RETURN(errctx);
}

static akerr_ErrorContext *match_identifier(akbasic_Runtime *obj)
{
    PREPARE_ERROR(errctx);
    char lexeme[AKBASIC_MAX_LINE_LENGTH];
    char basename[AKBASIC_MAX_LINE_LENGTH];
    const akbasic_Verb *verb = NULL;
    void *fndef = NULL;
    bool userfunction = false;
    char c = '\0';
    size_t used = 0;

    obj->tokentype = AKBASIC_TOK_IDENTIFIER;
    while ( !is_at_end(obj) ) {
	(void)peek(obj, &c);
	if ( isdigit((unsigned char)c) || isalpha((unsigned char)c) ) {
	    obj->current += 1;
	    continue;
	}
	switch ( c ) {
	case '@':
	    obj->tokentype = AKBASIC_TOK_IDENTIFIER_STRUCT;
	    obj->current += 1;
	    break;
	case '$':
	    obj->tokentype = AKBASIC_TOK_IDENTIFIER_STRING;
	    obj->current += 1;
	    break;
	case '%':
	    obj->tokentype = AKBASIC_TOK_IDENTIFIER_FLOAT;
	    obj->current += 1;
	    break;
	case '#':
	    obj->tokentype = AKBASIC_TOK_IDENTIFIER_INT;
	    obj->current += 1;
	    break;
	default:
	    break;
	}
	break;
    }

    PASS(errctx, get_lexeme(obj, lexeme, sizeof(lexeme)));

    /*
     * The verb table is searched on the *base* name, with any type suffix
     * stripped. The reference searches with the suffix still attached
     * (basicscanner.go:349), so `PRINT$` misses every table, the "Reserved word
     * in variable name" branch below is dead code, and `PRINT$ = 1` is quietly
     * accepted as an ordinary string variable. TODO.md section 6 item 16.
     */
    strncpy(basename, lexeme, sizeof(basename) - 1);
    basename[sizeof(basename) - 1] = '\0';
    used = strlen(basename);
    if ( obj->tokentype != AKBASIC_TOK_IDENTIFIER && used > 0 ) {
	basename[used - 1] = '\0';
    }
    PASS(errctx, akbasic_verb_lookup(basename, &verb));

    ATTEMPT {
	CATCH(errctx, akbasic_environment_get_function(obj->environment, lexeme, &fndef));
	userfunction = true;
    } CLEANUP {
    } PROCESS(errctx) {
    } HANDLE(errctx, AKERR_KEY) {
	userfunction = false;
    } FINISH(errctx, true);

    if ( obj->tokentype == AKBASIC_TOK_IDENTIFIER ) {
	if ( verb != NULL ) {
	    obj->tokentype = verb->tokentype;
	} else if ( userfunction ) {
	    obj->tokentype = AKBASIC_TOK_FUNCTION;
	}
    } else if ( verb != NULL ) {
	/*
	 * A suffixed identifier that collides with a verb or function name.
	 * PRINT$ is not a variable.
	 */
	PASS(errctx, akbasic_runtime_error(obj, AKBASIC_ERRCLASS_SYNTAX,
					   "Reserved word in variable name\n"));
	obj->hasError = true;
    }
    SUCCEED_RETURN(errctx);
}

akerr_ErrorContext *akbasic_scanner_scan(akbasic_Runtime *obj, const char *line, char *dest, size_t len)
{
    PREPARE_ERROR(errctx);
    char lexeme[AKBASIC_MAX_LINE_LENGTH];
    char c = '\0';
    bool done = false;

    FAIL_ZERO_RETURN(errctx, (obj != NULL), AKERR_NULLPOINTER, "NULL runtime in scan");
    FAIL_ZERO_RETURN(errctx, (line != NULL), AKERR_NULLPOINTER, "NULL line in scan");
    FAIL_ZERO_RETURN(errctx, (strlen(line) < AKBASIC_MAX_LINE_LENGTH), AKBASIC_ERR_BOUNDS,
		     "Source line of %zu characters exceeds the %d character limit",
		     strlen(line), AKBASIC_MAX_LINE_LENGTH - 1);

    strncpy(obj->line, line, AKBASIC_MAX_LINE_LENGTH - 1);
    obj->line[AKBASIC_MAX_LINE_LENGTH - 1] = '\0';
    PASS(errctx, akbasic_environment_zero_parser(obj->environment));
    obj->current = 0;
    obj->start = 0;
    obj->hasError = false;
    /*
     * Cleared here rather than by each caller, so the flag always describes the
     * line this call just scanned. It used to be cleared only in
     * process_line_repl(); the loading paths never touched it, which was
     * harmless while they ignored it and is not now that they decide a line's
     * number by it.
     */
    obj->hadlinenumber = false;

    while ( !is_at_end(obj) && !done ) {
	c = obj->line[obj->current];
	obj->current += 1;

	switch ( c ) {
	case '@': obj->tokentype = AKBASIC_TOK_ATSYMBOL;			break;
	case '^': obj->tokentype = AKBASIC_TOK_CARAT;			break;
	case '(': obj->tokentype = AKBASIC_TOK_LEFT_PAREN;		break;
	case ')': obj->tokentype = AKBASIC_TOK_RIGHT_PAREN;		break;
	case '+': obj->tokentype = AKBASIC_TOK_PLUS;			break;
	/*
	 * `->` reaches a field through a pointer, so a minus has to look ahead
	 * one character before it can call itself a minus. Nothing else in the
	 * language begins `->`, and `A# - >` is not an expression, so there is
	 * no spelling this makes ambiguous.
	 */
	case '-':
	    (void)match_next_char(obj, '>', AKBASIC_TOK_ARROW, AKBASIC_TOK_MINUS);
	    break;
	case '/': obj->tokentype = AKBASIC_TOK_LEFT_SLASH;		break;
	case '*': obj->tokentype = AKBASIC_TOK_STAR;			break;
	case ',': obj->tokentype = AKBASIC_TOK_COMMA;			break;
	case ':': obj->tokentype = AKBASIC_TOK_COLON;			break;
	/*
	 * A field separator. It only reaches this switch when it is *not* part
	 * of a number: match_number() is entered on a digit and consumes its own
	 * decimal point, so `1.5` never gets here and `P@.X#` always does.
	 */
	case '.': obj->tokentype = AKBASIC_TOK_DOT;			break;
	/*
	 * MOVSPR's two separators. Both were UNKNOWN TOKEN until sprites, so
	 * scanning them breaks nothing -- and a `#` only reaches this switch
	 * when it is not an identifier's type suffix, which match_identifier
	 * consumes before returning.
	 */
	case ';': obj->tokentype = AKBASIC_TOK_SEMICOLON;		break;
	case '#': obj->tokentype = AKBASIC_TOK_HASHMARK;		break;
	case '[': obj->tokentype = AKBASIC_TOK_LEFT_SQUAREBRACKET;	break;
	case ']': obj->tokentype = AKBASIC_TOK_RIGHT_SQUAREBRACKET;	break;
	case '=':
	    (void)match_next_char(obj, '=', AKBASIC_TOK_EQUAL, AKBASIC_TOK_ASSIGNMENT);
	    break;
	case '<':
	    if ( !match_next_char(obj, '=', AKBASIC_TOK_LESS_THAN_EQUAL, AKBASIC_TOK_LESS_THAN) ) {
		(void)match_next_char(obj, '>', AKBASIC_TOK_NOT_EQUAL, AKBASIC_TOK_LESS_THAN);
	    }
	    break;
	case '>':
	    (void)match_next_char(obj, '=', AKBASIC_TOK_GREATER_THAN_EQUAL, AKBASIC_TOK_GREATER_THAN);
	    break;
	case '"':
	    obj->start = obj->current;
	    PASS(errctx, match_string(obj));
	    break;
	case '\t':
	case ' ':
	    obj->start = obj->current;
	    break;
	case '\r':
	case '\n':
	    done = true;
	    break;
	default:
	    if ( isdigit((unsigned char)c) ) {
		PASS(errctx, match_number(obj));
	    } else if ( isalpha((unsigned char)c) ) {
		PASS(errctx, match_identifier(obj));
	    } else {
		char message[AKBASIC_MAX_LINE_LENGTH];
		snprintf(message, sizeof(message), "UNKNOWN TOKEN %c\n", c);
		PASS(errctx, akbasic_runtime_error(obj, AKBASIC_ERRCLASS_PARSE, message));
		obj->hasError = true;
		obj->start = obj->current;
	    }
	    break;
	}

	if ( done ) {
	    break;
	}
	if ( obj->tokentype != AKBASIC_TOK_UNDEFINED && !obj->hasError ) {
	    if ( obj->tokentype == AKBASIC_TOK_REM ) {
		/* Everything after REM is a comment. Stop, keeping the line intact. */
		break;
	    } else if ( obj->tokentype == AKBASIC_TOK_LINE_NUMBER ) {
		obj->hadlinenumber = true;
		/*
		 * The line number is not kept as a token. Rewrite the line to
		 * everything after it, minus leading spaces, and restart the
		 * cursor -- the REPL reads the rewritten line back out and
		 * stores *that* as the program text.
		 */
		int skip = obj->current;
		while ( obj->line[skip] == ' ' ) {
		    skip += 1;
		}
		memmove(obj->line, obj->line + skip, strlen(obj->line + skip) + 1);
		obj->current = 0;
	    } else {
		PASS(errctx, get_lexeme(obj, lexeme, sizeof(lexeme)));
		PASS(errctx, add_token(obj, obj->tokentype, lexeme));
		if ( obj->tokentype == AKBASIC_TOK_LITERAL_STRING ) {
		    /* Scanning stopped on the closing quote; step past it. */
		    obj->current += 1;
		}
	    }
	    obj->tokentype = AKBASIC_TOK_UNDEFINED;
	    obj->start = obj->current;
	}
    }

    if ( dest != NULL ) {
	FAIL_ZERO_RETURN(errctx, (strlen(obj->line) < len), AKBASIC_ERR_BOUNDS,
			 "Scanned line does not fit the caller's buffer");
	strncpy(dest, obj->line, len - 1);
	dest[len - 1] = '\0';
    }
    SUCCEED_RETURN(errctx);
}