/* ** 2001 September 15 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* ** An tokenizer for SQL ** ** This file contains C code that splits an SQL input string up into ** individual tokens and sends those tokens one-by-one over to the ** parser for analysis. ** ** $Id: tokenize.c,v 1.1.1.1 2006/02/03 20:35:19 hoganrobert Exp $ */ #include "sqliteInt.h" #include "os.h" #include #include /* ** The sqlite3KeywordCode function looks up an identifier to determine if ** it is a keyword. If it is a keyword, the token code of that keyword is ** returned. If the input is not a keyword, TK_ID is returned. ** ** The implementation of this routine was generated by a program, ** mkkeywordhash.h, located in the tool subdirectory of the distribution. ** The output of the mkkeywordhash.c program is written into a file ** named keywordhash.h and then included into this source file by ** the #include below. */ #include "keywordhash.h" /* ** If X is a character that can be used in an identifier and ** X&0x80==0 then isIdChar[X] will be 1. If X&0x80==0x80 then ** X is always an identifier character. (Hence all UTF-8 ** characters can be part of an identifier). isIdChar[X] will ** be 0 for every character in the lower 128 ASCII characters ** that cannot be used as part of an identifier. ** ** In this implementation, an identifier can be a string of ** alphabetic characters, digits, and "_" plus any character ** with the high-order bit set. The latter rule means that ** any sequence of UTF-8 characters or characters taken from ** an extended ISO8859 character set can form an identifier. ** ** Ticket #1066. the SQL standard does not allow '$' in the ** middle of identfiers. But many SQL implementations do. ** SQLite will allow '$' in identifiers for compatibility. ** But the feature is undocumented. */ static const char isIdChar[] = { /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ }; #define IdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && isIdChar[c-0x20])) /* ** Return the length of the token that begins at z[0]. ** Store the token type in *tokenType before returning. */ static int getToken(const unsigned char *z, int *tokenType){ int i, c; switch( *z ){ case ' ': case '\t': case '\n': case '\f': case '\r': { for(i=1; isspace(z[i]); i++){} *tokenType = TK_SPACE; return i; } case '-': { if( z[1]=='-' ){ for(i=2; (c=z[i])!=0 && c!='\n'; i++){} *tokenType = TK_COMMENT; return i; } *tokenType = TK_MINUS; return 1; } case '(': { *tokenType = TK_LP; return 1; } case ')': { *tokenType = TK_RP; return 1; } case ';': { *tokenType = TK_SEMI; return 1; } case '+': { *tokenType = TK_PLUS; return 1; } case '*': { *tokenType = TK_STAR; return 1; } case '/': { if( z[1]!='*' || z[2]==0 ){ *tokenType = TK_SLASH; return 1; } for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} if( c ) i++; *tokenType = TK_COMMENT; return i; } case '%': { *tokenType = TK_REM; return 1; } case '=': { *tokenType = TK_EQ; return 1 + (z[1]=='='); } case '<': { if( (c=z[1])=='=' ){ *tokenType = TK_LE; return 2; }else if( c=='>' ){ *tokenType = TK_NE; return 2; }else if( c=='<' ){ *tokenType = TK_LSHIFT; return 2; }else{ *tokenType = TK_LT; return 1; } } case '>': { if( (c=z[1])=='=' ){ *tokenType = TK_GE; return 2; }else if( c=='>' ){ *tokenType = TK_RSHIFT; return 2; }else{ *tokenType = TK_GT; return 1; } } case '!': { if( z[1]!='=' ){ *tokenType = TK_ILLEGAL; return 2; }else{ *tokenType = TK_NE; return 2; } } case '|': { if( z[1]!='|' ){ *tokenType = TK_BITOR; return 1; }else{ *tokenType = TK_CONCAT; return 2; } } case ',': { *tokenType = TK_COMMA; return 1; } case '&': { *tokenType = TK_BITAND; return 1; } case '~': { *tokenType = TK_BITNOT; return 1; } case '#': { for(i=1; isdigit(z[i]) || (i==1 && z[1]=='-'); i++){} *tokenType = TK_REGISTER; return i; } case '\'': case '"': { int delim = z[0]; for(i=1; (c=z[i])!=0; i++){ if( c==delim ){ if( z[i+1]==delim ){ i++; }else{ break; } } } if( c ) i++; *tokenType = TK_STRING; return i; } case '.': { *tokenType = TK_DOT; return 1; } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { *tokenType = TK_INTEGER; for(i=1; isdigit(z[i]); i++){} #ifndef SQLITE_OMIT_FLOATING_POINT if( z[i]=='.' && isdigit(z[i+1]) ){ i += 2; while( isdigit(z[i]) ){ i++; } *tokenType = TK_FLOAT; } if( (z[i]=='e' || z[i]=='E') && ( isdigit(z[i+1]) || ((z[i+1]=='+' || z[i+1]=='-') && isdigit(z[i+2])) ) ){ i += 2; while( isdigit(z[i]) ){ i++; } *tokenType = TK_FLOAT; } #endif return i; } case '[': { for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} *tokenType = TK_ID; return i; } case '?': { *tokenType = TK_VARIABLE; for(i=1; isdigit(z[i]); i++){} return i; } case ':': { for(i=1; IdChar(z[i]); i++){} *tokenType = i>1 ? TK_VARIABLE : TK_ILLEGAL; return i; } #ifndef SQLITE_OMIT_TCL_VARIABLE case '$': { *tokenType = TK_VARIABLE; if( z[1]=='{' ){ int nBrace = 1; for(i=2; (c=z[i])!=0 && nBrace; i++){ if( c=='{' ){ nBrace++; }else if( c=='}' ){ nBrace--; } } if( c==0 ) *tokenType = TK_ILLEGAL; }else{ int n = 0; for(i=1; (c=z[i])!=0; i++){ if( isalnum(c) || c=='_' ){ n++; }else if( c=='(' && n>0 ){ do{ i++; }while( (c=z[i])!=0 && !isspace(c) && c!=')' ); if( c==')' ){ i++; }else{ *tokenType = TK_ILLEGAL; } break; }else if( c==':' && z[i+1]==':' ){ i++; }else{ break; } } if( n==0 ) *tokenType = TK_ILLEGAL; } return i; } #endif #ifndef SQLITE_OMIT_BLOB_LITERAL case 'x': case 'X': { if( (c=z[1])=='\'' || c=='"' ){ int delim = c; *tokenType = TK_BLOB; for(i=2; (c=z[i])!=0; i++){ if( c==delim ){ if( i%2 ) *tokenType = TK_ILLEGAL; break; } if( !isxdigit(c) ){ *tokenType = TK_ILLEGAL; return i; } } if( c ) i++; return i; } /* Otherwise fall through to the next case */ } #endif default: { if( !IdChar(*z) ){ break; } for(i=1; IdChar(z[i]); i++){} *tokenType = keywordCode((char*)z, i); return i; } } *tokenType = TK_ILLEGAL; return 1; } int sqlite3GetToken(const unsigned char *z, int *tokenType){ return getToken(z, tokenType); } /* ** Run the parser on the given SQL string. The parser structure is ** passed in. An SQLITE_ status code is returned. If an error occurs ** and pzErrMsg!=NULL then an error message might be written into ** memory obtained from malloc() and *pzErrMsg made to point to that ** error message. Or maybe not. */ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ int nErr = 0; int i; void *pEngine; int tokenType; int lastTokenParsed = -1; sqlite3 *db = pParse->db; extern void *sqlite3ParserAlloc(void*(*)(int)); extern void sqlite3ParserFree(void*, void(*)(void*)); extern int sqlite3Parser(void*, int, Token, Parse*); db->flags &= ~SQLITE_Interrupt; pParse->rc = SQLITE_OK; i = 0; pEngine = sqlite3ParserAlloc((void*(*)(int))sqlite3MallocX); if( pEngine==0 ){ sqlite3SetString(pzErrMsg, "out of memory", (char*)0); return SQLITE_NOMEM; } assert( pParse->sLastToken.dyn==0 ); assert( pParse->pNewTable==0 ); assert( pParse->pNewTrigger==0 ); assert( pParse->nVar==0 ); assert( pParse->nVarExpr==0 ); assert( pParse->nVarExprAlloc==0 ); assert( pParse->apVarExpr==0 ); pParse->zTail = pParse->zSql = zSql; while( sqlite3_malloc_failed==0 && zSql[i]!=0 ){ assert( i>=0 ); pParse->sLastToken.z = &zSql[i]; assert( pParse->sLastToken.dyn==0 ); pParse->sLastToken.n = getToken((unsigned char*)&zSql[i],&tokenType); i += pParse->sLastToken.n; switch( tokenType ){ case TK_SPACE: case TK_COMMENT: { if( (db->flags & SQLITE_Interrupt)!=0 ){ pParse->rc = SQLITE_INTERRUPT; sqlite3SetString(pzErrMsg, "interrupt", (char*)0); goto abort_parse; } break; } case TK_ILLEGAL: { if( pzErrMsg ){ sqliteFree(*pzErrMsg); *pzErrMsg = sqlite3MPrintf("unrecognized token: \"%T\"", &pParse->sLastToken); } nErr++; goto abort_parse; } case TK_SEMI: { pParse->zTail = &zSql[i]; /* Fall thru into the default case */ } default: { sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); lastTokenParsed = tokenType; if( pParse->rc!=SQLITE_OK ){ goto abort_parse; } break; } } } abort_parse: if( zSql[i]==0 && nErr==0 && pParse->rc==SQLITE_OK ){ if( lastTokenParsed!=TK_SEMI ){ sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); pParse->zTail = &zSql[i]; } sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); } sqlite3ParserFree(pEngine, sqlite3FreeX); if( sqlite3_malloc_failed ){ pParse->rc = SQLITE_NOMEM; } if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ sqlite3SetString(&pParse->zErrMsg, sqlite3ErrStr(pParse->rc), (char*)0); } if( pParse->zErrMsg ){ if( pzErrMsg && *pzErrMsg==0 ){ *pzErrMsg = pParse->zErrMsg; }else{ sqliteFree(pParse->zErrMsg); } pParse->zErrMsg = 0; if( !nErr ) nErr++; } if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ sqlite3VdbeDelete(pParse->pVdbe); pParse->pVdbe = 0; } sqlite3DeleteTable(pParse->db, pParse->pNewTable); sqlite3DeleteTrigger(pParse->pNewTrigger); sqliteFree(pParse->apVarExpr); if( nErr>0 && (pParse->rc==SQLITE_OK || pParse->rc==SQLITE_DONE) ){ pParse->rc = SQLITE_ERROR; } return nErr; } /* The sqlite3_complete() API may be omitted (to save code space) by ** defining the following symbol. */ #ifndef SQLITE_OMIT_COMPLETE /* ** Token types used by the sqlite3_complete() routine. See the header ** comments on that procedure for additional information. */ #define tkSEMI 0 #define tkWS 1 #define tkOTHER 2 #define tkEXPLAIN 3 #define tkCREATE 4 #define tkTEMP 5 #define tkTRIGGER 6 #define tkEND 7 /* ** Return TRUE if the given SQL string ends in a semicolon. ** ** Special handling is require for CREATE TRIGGER statements. ** Whenever the CREATE TRIGGER keywords are seen, the statement ** must end with ";END;". ** ** This implementation uses a state machine with 7 states: ** ** (0) START At the beginning or end of an SQL statement. This routine ** returns 1 if it ends in the START state and 0 if it ends ** in any other state. ** ** (1) NORMAL We are in the middle of statement which ends with a single ** semicolon. ** ** (2) EXPLAIN The keyword EXPLAIN has been seen at the beginning of ** a statement. ** ** (3) CREATE The keyword CREATE has been seen at the beginning of a ** statement, possibly preceeded by EXPLAIN and/or followed by ** TEMP or TEMPORARY ** ** (4) TRIGGER We are in the middle of a trigger definition that must be ** ended by a semicolon, the keyword END, and another semicolon. ** ** (5) SEMI We've seen the first semicolon in the ";END;" that occurs at ** the end of a trigger definition. ** ** (6) END We've seen the ";END" of the ";END;" that occurs at the end ** of a trigger difinition. ** ** Transitions between states above are determined by tokens extracted ** from the input. The following tokens are significant: ** ** (0) tkSEMI A semicolon. ** (1) tkWS Whitespace ** (2) tkOTHER Any other SQL token. ** (3) tkEXPLAIN The "explain" keyword. ** (4) tkCREATE The "create" keyword. ** (5) tkTEMP The "temp" or "temporary" keyword. ** (6) tkTRIGGER The "trigger" keyword. ** (7) tkEND The "end" keyword. ** ** Whitespace never causes a state transition and is always ignored. ** ** If we compile with SQLITE_OMIT_TRIGGER, all of the computation needed ** to recognize the end of a trigger can be omitted. All we have to do ** is look for a semicolon that is not part of an string or comment. */ int sqlite3_complete(const char *zSql){ u8 state = 0; /* Current state, using numbers defined in header comment */ u8 token; /* Value of the next token */ #ifndef SQLITE_OMIT_TRIGGER /* A complex statement machine used to detect the end of a CREATE TRIGGER ** statement. This is the normal case. */ static const u8 trans[7][8] = { /* Token: */ /* State: ** SEMI WS OTHER EXPLAIN CREATE TEMP TRIGGER END */ /* 0 START: */ { 0, 0, 1, 2, 3, 1, 1, 1, }, /* 1 NORMAL: */ { 0, 1, 1, 1, 1, 1, 1, 1, }, /* 2 EXPLAIN: */ { 0, 2, 1, 1, 3, 1, 1, 1, }, /* 3 CREATE: */ { 0, 3, 1, 1, 1, 3, 4, 1, }, /* 4 TRIGGER: */ { 5, 4, 4, 4, 4, 4, 4, 4, }, /* 5 SEMI: */ { 5, 5, 4, 4, 4, 4, 4, 6, }, /* 6 END: */ { 0, 6, 4, 4, 4, 4, 4, 4, }, }; #else /* If triggers are not suppored by this compile then the statement machine ** used to detect the end of a statement is much simplier */ static const u8 trans[2][3] = { /* Token: */ /* State: ** SEMI WS OTHER */ /* 0 START: */ { 0, 0, 1, }, /* 1 NORMAL: */ { 0, 1, 1, }, }; #endif /* SQLITE_OMIT_TRIGGER */ while( *zSql ){ switch( *zSql ){ case ';': { /* A semicolon */ token = tkSEMI; break; } case ' ': case '\r': case '\t': case '\n': case '\f': { /* White space is ignored */ token = tkWS; break; } case '/': { /* C-style comments */ if( zSql[1]!='*' ){ token = tkOTHER; break; } zSql += 2; while( zSql[0] && (zSql[0]!='*' || zSql[1]!='/') ){ zSql++; } if( zSql[0]==0 ) return 0; zSql++; token = tkWS; break; } case '-': { /* SQL-style comments from "--" to end of line */ if( zSql[1]!='-' ){ token = tkOTHER; break; } while( *zSql && *zSql!='\n' ){ zSql++; } if( *zSql==0 ) return state==0; token = tkWS; break; } case '[': { /* Microsoft-style identifiers in [...] */ zSql++; while( *zSql && *zSql!=']' ){ zSql++; } if( *zSql==0 ) return 0; token = tkOTHER; break; } case '"': /* single- and double-quoted strings */ case '\'': { int c = *zSql; zSql++; while( *zSql && *zSql!=c ){ zSql++; } if( *zSql==0 ) return 0; token = tkOTHER; break; } default: { int c; if( IdChar((u8)*zSql) ){ /* Keywords and unquoted identifiers */ int nId; for(nId=1; IdChar(zSql[nId]); nId++){} #ifdef SQLITE_OMIT_TRIGGER token = tkOTHER; #else switch( *zSql ){ case 'c': case 'C': { if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){ token = tkCREATE; }else{ token = tkOTHER; } break; } case 't': case 'T': { if( nId==7 && sqlite3StrNICmp(zSql, "trigger", 7)==0 ){ token = tkTRIGGER; }else if( nId==4 && sqlite3StrNICmp(zSql, "temp", 4)==0 ){ token = tkTEMP; }else if( nId==9 && sqlite3StrNICmp(zSql, "temporary", 9)==0 ){ token = tkTEMP; }else{ token = tkOTHER; } break; } case 'e': case 'E': { if( nId==3 && sqlite3StrNICmp(zSql, "end", 3)==0 ){ token = tkEND; }else #ifndef SQLITE_OMIT_EXPLAIN if( nId==7 && sqlite3StrNICmp(zSql, "explain", 7)==0 ){ token = tkEXPLAIN; }else #endif { token = tkOTHER; } break; } default: { token = tkOTHER; break; } } #endif /* SQLITE_OMIT_TRIGGER */ zSql += nId-1; }else{ /* Operators and special symbols */ token = tkOTHER; } break; } } state = trans[state][token]; zSql++; } return state==0; } #ifndef SQLITE_OMIT_UTF16 /* ** This routine is the same as the sqlite3_complete() routine described ** above, except that the parameter is required to be UTF-16 encoded, not ** UTF-8. */ int sqlite3_complete16(const void *zSql){ sqlite3_value *pVal; char const *zSql8; int rc = 0; pVal = sqlite3ValueNew(); sqlite3ValueSetStr(pVal, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC); zSql8 = sqlite3ValueText(pVal, SQLITE_UTF8); if( zSql8 ){ rc = sqlite3_complete(zSql8); } sqlite3ValueFree(pVal); return rc; } #endif /* SQLITE_OMIT_UTF16 */ #endif /* SQLITE_OMIT_COMPLETE */