xref: /trunk/main/basic/source/comp/token.cxx (revision e1f63238)
1 /**************************************************************
2  *
3  * Licensed to the Apache Software Foundation (ASF) under one
4  * or more contributor license agreements.  See the NOTICE file
5  * distributed with this work for additional information
6  * regarding copyright ownership.  The ASF licenses this file
7  * to you under the Apache License, Version 2.0 (the
8  * "License"); you may not use this file except in compliance
9  * with the License.  You may obtain a copy of the License at
10  *
11  *   http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing,
14  * software distributed under the License is distributed on an
15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16  * KIND, either express or implied.  See the License for the
17  * specific language governing permissions and limitations
18  * under the License.
19  *
20  *************************************************************/
21 
22 
23 
24 // MARKER(update_precomp.py): autogen include statement, do not remove
25 #include "precompiled_basic.hxx"
26 
27 #include <ctype.h>
28 #include "sbcomp.hxx"
29 
30 struct TokenTable { SbiToken t; const char *s; };
31 
32 static short nToken;					// Anzahl der Tokens
33 
34 static TokenTable* pTokTable;
35 
36 static TokenTable aTokTable_Basic [] = {		// Token-Tabelle:
37 
38 	{ CAT,		"&" },
39 	{ MUL,		"*" },
40 	{ PLUS,		"+" },
41 	{ MINUS,	"-" },
42 	{ DIV,		"/" },
43 	{ EOS,		":" },
44 	{ ASSIGN,	":=" },
45 	{ LT,		"<" },
46 	{ LE,		"<=" },
47 	{ NE,		"<>" },
48 	{ EQ,		"=" },
49 	{ GT,		">" },
50 	{ GE,		">=" },
51 	{ ACCESS,	"Access" },
52 	{ ALIAS,	"Alias" },
53 	{ AND,		"And" },
54 	{ ANY,		"Any" },
55 	{ APPEND,	"Append" },
56 	{ AS,		"As" },
57 	{ BASE,		"Base" },
58 	{ BINARY,	"Binary" },
59 	{ TBOOLEAN,	"Boolean" },
60 	{ BYREF,	"ByRef", },
61 	{ TBYTE,	"Byte", },
62 	{ BYVAL,	"ByVal", },
63 	{ CALL,		"Call" },
64 	{ CASE,		"Case" },
65 	{ _CDECL_,	"Cdecl" },
66 	{ CLASSMODULE, "ClassModule" },
67 	{ CLOSE,	"Close" },
68 	{ COMPARE,	"Compare" },
69 	{ COMPATIBLE,"Compatible" },
70 	{ _CONST_,	"Const" },
71 	{ TCURRENCY,"Currency" },
72 	{ TDATE,	"Date" },
73 	{ DECLARE,	"Declare" },
74 	{ DEFBOOL,	"DefBool" },
75 	{ DEFCUR,	"DefCur" },
76 	{ DEFDATE,	"DefDate" },
77 	{ DEFDBL,	"DefDbl" },
78 	{ DEFERR,	"DefErr" },
79 	{ DEFINT,	"DefInt" },
80 	{ DEFLNG,	"DefLng" },
81 	{ DEFOBJ,	"DefObj" },
82 	{ DEFSNG,	"DefSng" },
83 	{ DEFSTR,	"DefStr" },
84 	{ DEFVAR,	"DefVar" },
85 	{ DIM,		"Dim" },
86 	{ DO,		"Do" },
87 	{ TDOUBLE,	"Double" },
88 	{ EACH,		"Each" },
89 	{ ELSE,		"Else" },
90 	{ ELSEIF,	"ElseIf" },
91 	{ END,		"End" },
92 	{ ENDENUM,	"End Enum" },
93 	{ ENDFUNC,	"End Function" },
94 	{ ENDIF,	"End If" },
95 	{ ENDPROPERTY, "End Property" },
96 	{ ENDSELECT,"End Select" },
97 	{ ENDSUB,	"End Sub" },
98 	{ ENDTYPE,	"End Type" },
99 	{ ENDIF,	"EndIf" },
100 	{ ENUM,		"Enum" },
101 	{ EQV,		"Eqv" },
102 	{ ERASE,	"Erase" },
103 	{ _ERROR_,	"Error" },
104 	{ EXIT,		"Exit" },
105 	{ EXPLICIT,	"Explicit" },
106 	{ FOR,		"For" },
107 	{ FUNCTION,	"Function" },
108 	{ GET,		"Get" },
109 	{ GLOBAL,	"Global" },
110 	{ GOSUB,	"GoSub" },
111 	{ GOTO,		"GoTo" },
112 	{ IF,		"If" },
113 	{ IMP,		"Imp" },
114 	{ IMPLEMENTS, "Implements" },
115 	{ _IN_,		"In" },
116 	{ INPUT,	"Input" },				// auch INPUT #
117 	{ TINTEGER,	"Integer" },
118 	{ IS,		"Is" },
119 	{ LET,		"Let" },
120 	{ LIB,		"Lib" },
121 	{ LIKE,		"Like" },
122 	{ LINE,		"Line" },
123 	{ LINEINPUT,"Line Input" },
124 	{ LOCAL,	"Local" },
125 	{ LOCK,		"Lock" },
126 	{ TLONG,	"Long" },
127 	{ LOOP,		"Loop" },
128 	{ LPRINT,	"LPrint" },
129 	{ LSET,     "LSet" }, // JSM
130 	{ MOD,		"Mod" },
131 	{ NAME,		"Name" },
132 	{ NEW,		"New" },
133 	{ NEXT,		"Next" },
134 	{ NOT,		"Not" },
135 	{ TOBJECT,	"Object" },
136 	{ ON,		"On" },
137 	{ OPEN,		"Open" },
138 	{ OPTION,	"Option" },
139 	{ _OPTIONAL_,	"Optional" },
140 	{ OR,		"Or" },
141 	{ OUTPUT,	"Output" },
142 	{ PARAMARRAY,	"ParamArray" },
143 	{ PRESERVE, "Preserve" },
144 	{ PRINT,	"Print" },
145 	{ PRIVATE,	"Private" },
146 	{ PROPERTY,	"Property" },
147 	{ PUBLIC,	"Public" },
148 	{ RANDOM,	"Random" },
149 	{ READ,		"Read" },
150 	{ REDIM,	"ReDim" },
151 	{ REM, 		"Rem" },
152 	{ RESUME,	"Resume" },
153 	{ RETURN,	"Return" },
154 	{ RSET,     "RSet" }, // JSM
155 	{ SELECT,	"Select" },
156 	{ SET,		"Set" },
157 #ifdef SHARED
158 #undef SHARED
159 #define tmpSHARED
160 #endif
161 	{ SHARED,	"Shared" },
162 #ifdef tmpSHARED
163 #define SHARED
164 #undef tmpSHARED
165 #endif
166 	{ TSINGLE,	"Single" },
167 	{ STATIC,	"Static" },
168 	{ STEP,		"Step" },
169 	{ STOP,		"Stop" },
170 	{ TSTRING,	"String" },
171 	{ SUB,		"Sub" },
172 	{ STOP,		"System" },
173 	{ TEXT,		"Text" },
174 	{ THEN,		"Then" },
175 	{ TO,		"To", },
176 	{ TYPE,		"Type" },
177 	{ TYPEOF,	"TypeOf" },
178 	{ UNTIL,	"Until" },
179 	{ TVARIANT,	"Variant" },
180 	{ VBASUPPORT,	"VbaSupport" },
181 	{ WEND,		"Wend" },
182 	{ WHILE,	"While" },
183 	{ WITH,		"With" },
184 	{ WITHEVENTS,	"WithEvents" },
185 	{ WRITE,	"Write" },				// auch WRITE #
186 	{ XOR,		"Xor" },
187 	{ NIL,		"" }
188 };
189 
190 /*
191 TokenTable aTokTable_Java [] = {		// Token-Tabelle:
192 
193 	{ JS_LOG_NOT,	"!" },
194 	{ JS_NE,		"!=" },
195 	{ JS_MOD,		"%" },
196 	{ JS_ASS_MOD,	"%=" },
197 	{ JS_BIT_AND,	"&" },
198 	{ JS_LOG_AND,	"&&" },
199 	{ JS_ASS_AND,	"&=" },
200 	{ JS_LPAREN,	"(" },
201 	{ JS_RPAREN,	")" },
202 	{ JS_MUL,		"*" },
203 	{ JS_ASS_MUL,	"*=" },
204 	{ JS_PLUS,		"+" },
205 	{ JS_INC,		"++" },
206 	{ JS_ASS_PLUS,	"+=" },
207 	{ JS_COMMA,		"," },
208 	{ JS_MINUS,		"-" },
209 	{ JS_DEC,		"--" },
210 	{ JS_ASS_MINUS,	"-=" },
211 	{ JS_DIV,		"/" },
212 	{ JS_ASS_DIV,	"/=" },
213 	{ JS_COND_SEL,	":" },
214 	{ JS_LT,		"<" },
215 	{ JS_LSHIFT,	"<<" },
216 	{ JS_ASS_LSHIFT,"<<=" },
217 	{ JS_LE,		"<=" },
218 	{ JS_NE,		"<>" },
219 	{ JS_ASSIGNMENT,"=" },
220 	{ JS_EQ,		"==" },
221 	{ JS_GT,		">" },
222 	{ JS_RSHIFT,	">>" },
223 	{ JS_ASS_RSHIFT,">>=" },
224 	{ JS_RSHIFT_Z,	">>>" },
225 	{ JS_ASS_RSHIFT_Z,">>>=" },
226 	{ JS_GE,		">=" },
227 	{ JS_COND_QUEST,"?" },
228 	{ ACCESS,	"Access" },
229 	{ ALIAS,	"Alias" },
230 	{ AND,		"And" },
231 	{ ANY,		"Any" },
232 	{ APPEND,	"Append" },
233 	{ AS,		"As" },
234 	{ BASE,		"Base" },
235 	{ BINARY,	"Binary" },
236 	{ TBOOLEAN,	"Boolean" },
237 	{ BYVAL,	"ByVal", },
238 	{ CALL,		"Call" },
239 	{ CASE,		"Case" },
240 	{ _CDECL_,	"Cdecl" },
241 	{ CLOSE,	"Close" },
242 	{ COMPARE,	"Compare" },
243 	{ _CONST_,	"Const" },
244 	{ TCURRENCY,"Currency" },
245 	{ TDATE,	"Date" },
246 	{ DECLARE,	"Declare" },
247 	{ DEFBOOL,	"DefBool" },
248 	{ DEFCUR,	"DefCur" },
249 	{ DEFDATE,	"DefDate" },
250 	{ DEFDBL,	"DefDbl" },
251 	{ DEFERR,	"DefErr" },
252 	{ DEFINT,	"DefInt" },
253 	{ DEFLNG,	"DefLng" },
254 	{ DEFOBJ,	"DefObj" },
255 	{ DEFSNG,	"DefSng" },
256 	{ DEFSTR,	"DefStr" },
257 	{ DEFVAR,	"DefVar" },
258 	{ DIM,		"Dim" },
259 	{ DO,		"Do" },
260 	{ TDOUBLE,	"Double" },
261 	{ EACH,		"Each" },
262 	{ ELSE,		"Else" },
263 	{ ELSEIF,	"ElseIf" },
264 	{ END,		"End" },
265 	{ ENDFUNC,	"End Function" },
266 	{ ENDIF,	"End If" },
267 	{ ENDSELECT,"End Select" },
268 	{ ENDSUB,	"End Sub" },
269 	{ ENDTYPE,	"End Type" },
270 	{ ENDIF,	"EndIf" },
271 	{ EQV,		"Eqv" },
272 	{ ERASE,	"Erase" },
273 	{ _ERROR_,	"Error" },
274 	{ EXIT,		"Exit" },
275 	{ EXPLICIT,	"Explicit" },
276 	{ FOR,		"For" },
277 	{ FUNCTION,	"Function" },
278 	{ GLOBAL,	"Global" },
279 	{ GOSUB,	"GoSub" },
280 	{ GOTO,		"GoTo" },
281 	{ IF,		"If" },
282 	{ IMP,		"Imp" },
283 	{ _IN_,		"In" },
284 	{ INPUT,	"Input" },				// auch INPUT #
285 	{ TINTEGER,	"Integer" },
286 	{ IS,		"Is" },
287 	{ LET,		"Let" },
288 	{ LIB,		"Lib" },
289 	{ LINE,		"Line" },
290 	{ LINEINPUT,"Line Input" },
291 	{ LOCAL,	"Local" },
292 	{ LOCK,		"Lock" },
293 	{ TLONG,	"Long" },
294 	{ LOOP,		"Loop" },
295 	{ LPRINT,	"LPrint" },
296 	{ LSET,     "LSet" }, // JSM
297 	{ MOD,		"Mod" },
298 	{ NAME,		"Name" },
299 	{ NEW,		"New" },
300 	{ NEXT,		"Next" },
301 	{ NOT,		"Not" },
302 	{ TOBJECT,	"Object" },
303 	{ ON,		"On" },
304 	{ OPEN,		"Open" },
305 	{ OPTION,	"Option" },
306 	{ _OPTIONAL_,	"Optional" },
307 	{ OR,		"Or" },
308 	{ OUTPUT,	"Output" },
309 	{ PRESERVE, "Preserve" },
310 	{ PRINT,	"Print" },
311 	{ PRIVATE,	"Private" },
312 	{ PUBLIC,	"Public" },
313 	{ RANDOM,	"Random" },
314 	{ READ,		"Read" },
315 	{ REDIM,	"ReDim" },
316 	{ REM, 		"Rem" },
317 	{ RESUME,	"Resume" },
318 	{ RETURN,	"Return" },
319 	{ RSET,     "RSet" }, // JSM
320 	{ SELECT,	"Select" },
321 	{ SET,		"Set" },
322 	{ SHARED,	"Shared" },
323 	{ TSINGLE,	"Single" },
324 	{ STATIC,	"Static" },
325 	{ STEP,		"Step" },
326 	{ STOP,		"Stop" },
327 	{ TSTRING,	"String" },
328 	{ SUB,		"Sub" },
329 	{ STOP,		"System" },
330 	{ TEXT,		"Text" },
331 	{ THEN,		"Then" },
332 	{ TO,		"To", },
333 	{ TYPE,		"Type" },
334 	{ UNTIL,	"Until" },
335 	{ TVARIANT,	"Variant" },
336 	{ WEND,		"Wend" },
337 	{ WHILE,	"While" },
338 	{ WITH,		"With" },
339 	{ WRITE,	"Write" },				// auch WRITE #
340 	{ XOR,		"Xor" },
341 	{ JS_LINDEX,	"[" },
342 	{ JS_RINDEX,	"]" },
343 	{ JS_BIT_XOR,	"^" },
344 	{ JS_ASS_XOR,	"^=" },
345 	{ JS_BIT_OR,	"|" },
346 	{ JS_ASS_OR,	"|=" },
347 	{ JS_LOG_OR,	"||" },
348 	{ JS_BIT_NOT,	"~" },
349 	{ NIL }
350 };
351 */
352 
353 // #i109076
354 TokenLabelInfo::TokenLabelInfo( void )
355 {
356 	m_pTokenCanBeLabelTab = new bool[VBASUPPORT+1];
357 	for( int i = 0 ; i <= VBASUPPORT ; ++i )
358 		m_pTokenCanBeLabelTab[i] = false;
359 
360 	// Token accepted as label by VBA
361 	SbiToken eLabelToken[] = { ACCESS, ALIAS, APPEND, BASE, BINARY, CLASSMODULE,
362 		COMPARE, COMPATIBLE, DEFERR, _ERROR_, EXPLICIT, LIB, LINE, LPRINT, NAME,
363 		TOBJECT, OUTPUT, PROPERTY, RANDOM, READ, STEP, STOP, TEXT, VBASUPPORT, NIL };
364 	SbiToken* pTok = eLabelToken;
365 	SbiToken eTok;
366 	for( pTok = eLabelToken ; (eTok = *pTok) != NIL ; ++pTok )
367 		m_pTokenCanBeLabelTab[eTok] = true;
368 }
369 
370 TokenLabelInfo::~TokenLabelInfo()
371 {
372 	delete[] m_pTokenCanBeLabelTab;
373 }
374 
375 
376 // Der Konstruktor ermittelt die Laenge der Token-Tabelle.
377 
378 SbiTokenizer::SbiTokenizer( const ::rtl::OUString& rSrc, StarBASIC* pb )
379 		   : SbiScanner( rSrc, pb )
380 {
381 	pTokTable = aTokTable_Basic;
382 	//if( StarBASIC::GetGlobalLanguageMode() == SB_LANG_JAVASCRIPT )
383 	//	pTokTable = aTokTable_Java;
384 	TokenTable *tp;
385 	bEof = bAs = sal_False;
386 	eCurTok = NIL;
387 	ePush = NIL;
388 	bEos = bKeywords = bErrorIsSymbol = sal_True;
389 	if( !nToken )
390 		for( nToken = 0, tp = pTokTable; tp->t; nToken++, tp++ ) {}
391 }
392 
393 SbiTokenizer::~SbiTokenizer()
394 {
395 }
396 
397 // Wiederablage (Pushback) eines Tokens. (Bis zu 2 Tokens)
398 
399 void SbiTokenizer::Push( SbiToken t )
400 {
401 	if( ePush != NIL )
402 		Error( SbERR_INTERNAL_ERROR, "PUSH" );
403 	else ePush = t;
404 }
405 
406 void SbiTokenizer::Error( SbError code, const char* pMsg )
407 {
408 	aError = String::CreateFromAscii( pMsg );
409 	Error( code );
410 }
411 
412 void SbiTokenizer::Error( SbError code, String aMsg )
413 {
414 	aError = aMsg;
415 	Error( code );
416 }
417 
418 void SbiTokenizer::Error( SbError code, SbiToken tok )
419 {
420 	aError = Symbol( tok );
421 	Error( code );
422 }
423 
424 // Einlesen des naechsten Tokens, ohne dass das Token geschluckt wird
425 
426 SbiToken SbiTokenizer::Peek()
427 {
428 	if( ePush == NIL )
429 	{
430 		sal_uInt16 nOldLine = nLine;
431 		sal_uInt16 nOldCol1 = nCol1;
432 		sal_uInt16 nOldCol2 = nCol2;
433 		ePush = Next();
434 		nPLine = nLine; nLine = nOldLine;
435 		nPCol1 = nCol1; nCol1 = nOldCol1;
436 		nPCol2 = nCol2; nCol2 = nOldCol2;
437 	}
438 	return eCurTok = ePush;
439 }
440 
441 // Dies ist fuer die Decompilation.
442 // Zahlen und Symbole liefern einen Leerstring zurueck.
443 
444 const String& SbiTokenizer::Symbol( SbiToken t )
445 {
446 	// Zeichen-Token?
447 	if( t < FIRSTKWD )
448 	{
449 		aSym = (char) t;
450 		return aSym;
451 	}
452 	switch( t )
453 	{
454 		case NEG   : aSym = '-'; return aSym;
455 		case EOS   : aSym = String::CreateFromAscii( ":/CRLF" ); return aSym;
456 		case EOLN  : aSym = String::CreateFromAscii( "CRLF" ); return aSym;
457 		default: break;
458 	}
459 	TokenTable* tp = pTokTable;
460 	for( short i = 0; i < nToken; i++, tp++ )
461 	{
462 		if( tp->t == t )
463 		{
464 			aSym = String::CreateFromAscii( tp->s );
465 			return aSym;
466 		}
467 	}
468 	const sal_Unicode *p = aSym.GetBuffer();
469 	if (*p <= ' ') aSym = String::CreateFromAscii( "???" );
470 	return aSym;
471 }
472 
473 // Einlesen des naechsten Tokens und Ablage desselben
474 // Tokens, die nicht in der Token-Tabelle vorkommen, werden
475 // direkt als Zeichen zurueckgeliefert.
476 // Einige Worte werden gesondert behandelt.
477 
478 SbiToken SbiTokenizer::Next()
479 {
480 	if (bEof) return EOLN;
481 	// Schon eines eingelesen?
482 	if( ePush != NIL )
483 	{
484 		eCurTok = ePush;
485 		ePush = NIL;
486 		nLine = nPLine;
487 		nCol1 = nPCol1;
488 		nCol2 = nPCol2;
489 		bEos = IsEoln( eCurTok );
490 		return eCurTok;
491 	}
492 	TokenTable *tp;
493 
494 	// Sonst einlesen:
495 	if( !NextSym() )
496 	{
497 		bEof = bEos = sal_True;
498 		return eCurTok = EOLN;
499 	}
500 	// Zeilenende?
501 	if( aSym.GetBuffer()[0] == '\n' )
502 	{
503 		bEos = sal_True; return eCurTok = EOLN;
504 	}
505 	bEos = sal_False;
506 
507 	// Zahl?
508 	if( bNumber )
509 		return eCurTok = NUMBER;
510 
511 	// String?
512 	else if( ( eScanType == SbxDATE || eScanType == SbxSTRING ) && !bSymbol )
513 		return eCurTok = FIXSTRING;
514 	// Sonderfaelle von Zeichen, die zwischen "Z" und "a" liegen. ICompare()
515 	// wertet die Position dieser Zeichen unterschiedlich aus.
516 	else if( aSym.GetBuffer()[0] == '^' )
517 		return eCurTok = EXPON;
518 	else if( aSym.GetBuffer()[0] == '\\' )
519 		return eCurTok = IDIV;
520 	else
521 	{
522 		// Mit Typkennung oder ein Symbol und keine Keyword-Erkennung?
523 		// Dann kein Token-Test
524 		if( eScanType != SbxVARIANT
525 		 || ( !bKeywords && bSymbol ) )
526 			return eCurTok = SYMBOL;
527 		// Gueltiges Token?
528 		short lb = 0;
529 		short ub = nToken-1;
530 		short delta;
531 		do
532 		{
533 			delta = (ub - lb) >> 1;
534 			tp = &pTokTable[ lb + delta ];
535 			StringCompare res = aSym.CompareIgnoreCaseToAscii( tp->s );
536 			// Gefunden?
537 			if( res == COMPARE_EQUAL )
538 				goto special;
539 			// Groesser? Dann untere Haelfte
540 			if( res == COMPARE_LESS )
541 			{
542 				if ((ub - lb) == 2) ub = lb;
543 				else ub = ub - delta;
544 			}
545 			// Kleiner? Dann obere Haelfte
546 			else
547 			{
548 				if ((ub -lb) == 2) lb = ub;
549 				else lb = lb + delta;
550 			}
551 		} while( delta );
552 		// Symbol? Wenn nicht >= Token
553 		sal_Unicode ch = aSym.GetBuffer()[0];
554 		if( !BasicSimpleCharClass::isAlpha( ch, bCompatible ) && !bSymbol )
555 			return eCurTok = (SbiToken) (ch & 0x00FF);
556 		return eCurTok = SYMBOL;
557 	}
558 special:
559 	// #i92642
560 	bool bStartOfLine = (eCurTok == NIL || eCurTok == REM || eCurTok == EOLN);
561 	if( !bStartOfLine && (tp->t == NAME || tp->t == LINE) )
562 		return eCurTok = SYMBOL;
563 	else if( tp->t == TEXT )
564 		return eCurTok = SYMBOL;
565 
566 	// #i92642: Special LINE token handling -> SbiParser::Line()
567 
568 	// END IF, CASE, SUB, DEF, FUNCTION, TYPE, CLASS, WITH
569 	if( tp->t == END )
570 	{
571 		// AB, 15.3.96, Spezialbehandlung fuer END, beim Peek() geht die
572 		// aktuelle Zeile verloren, daher alles merken und danach restaurieren
573 		sal_uInt16 nOldLine = nLine;
574 		sal_uInt16 nOldCol  = nCol;
575 		sal_uInt16 nOldCol1 = nCol1;
576 		sal_uInt16 nOldCol2 = nCol2;
577 		String aOldSym = aSym;
578 		SaveLine();				// pLine im Scanner sichern
579 
580 		eCurTok = Peek();
581 		switch( eCurTok )
582 		{
583 			case IF: 	   Next(); eCurTok = ENDIF; break;
584 			case SELECT:   Next(); eCurTok = ENDSELECT; break;
585 			case SUB: 	   Next(); eCurTok = ENDSUB; break;
586 			case FUNCTION: Next(); eCurTok = ENDFUNC; break;
587 			case PROPERTY: Next(); eCurTok = ENDPROPERTY; break;
588 			case TYPE: 	   Next(); eCurTok = ENDTYPE; break;
589 			case ENUM: 	   Next(); eCurTok = ENDENUM; break;
590 			case WITH: 	   Next(); eCurTok = ENDWITH; break;
591 			default : 	   eCurTok = END;
592 		}
593 		nCol1 = nOldCol1;
594 		if( eCurTok == END )
595 		{
596 			// Alles zuruecksetzen, damit Token nach END ganz neu gelesen wird
597 			ePush = NIL;
598 			nLine = nOldLine;
599 			nCol  = nOldCol;
600 			nCol2 = nOldCol2;
601 			aSym = aOldSym;
602 			RestoreLine();		// pLine im Scanner restaurieren
603 		}
604 		return eCurTok;
605 	}
606 	// Sind Datentypen Keywords?
607 	// Nur nach AS, sonst sind es Symbole!
608 	// Es gibt ja ERROR(), DATA(), STRING() etc.
609 	eCurTok = tp->t;
610 	// AS: Datentypen sind Keywords
611 	if( tp->t == AS )
612 		bAs = sal_True;
613 	else
614 	{
615 		if( bAs )
616 			bAs = sal_False;
617 		else if( eCurTok >= DATATYPE1 && eCurTok <= DATATYPE2 && (bErrorIsSymbol || eCurTok != _ERROR_) )
618 			eCurTok = SYMBOL;
619 	}
620 
621 	// CLASSMODULE, PROPERTY, GET, ENUM token only visible in compatible mode
622 	SbiToken eTok = tp->t;
623 	if( bCompatible )
624 	{
625 		// #129904 Suppress system
626 		if( eTok == STOP && aSym.CompareIgnoreCaseToAscii( "system" ) == COMPARE_EQUAL )
627 			eCurTok = SYMBOL;
628 
629 		if( eTok == GET && bStartOfLine )
630 			eCurTok = SYMBOL;
631 	}
632 	else
633 	{
634 		if( eTok == CLASSMODULE ||
635 			eTok == IMPLEMENTS ||
636 			eTok == PARAMARRAY ||
637 			eTok == ENUM ||
638 			eTok == PROPERTY ||
639 			eTok == GET ||
640 			eTok == TYPEOF )
641 		{
642 			eCurTok = SYMBOL;
643 		}
644 	}
645 
646 	bEos = IsEoln( eCurTok );
647 	return eCurTok;
648 }
649 
650 #ifdef _MSC_VER
651 #pragma optimize("",off)
652 #endif
653 
654 // Kann das aktuell eingelesene Token ein Label sein?
655 
656 sal_Bool SbiTokenizer::MayBeLabel( sal_Bool bNeedsColon )
657 {
658 	if( eCurTok == SYMBOL || m_aTokenLabelInfo.canTokenBeLabel( eCurTok ) )
659 		return bNeedsColon ? DoesColonFollow() : sal_True;
660 	else
661 		return sal_Bool( eCurTok == NUMBER
662 				  && eScanType == SbxINTEGER
663 				  && nVal >= 0 );
664 }
665 
666 #ifdef _MSC_VER
667 #pragma optimize("",off)
668 #endif
669 
670 
671 void SbiTokenizer::Hilite( SbTextPortions& rList )
672 {
673 	bErrors = sal_False;
674 	bUsedForHilite = sal_True;
675 	SbiToken eLastTok = NIL;
676 	for( ;; )
677 	{
678 		Next();
679 		if( IsEof() )
680 			break;
681 		SbTextPortion aRes;
682 		aRes.nLine = nLine;
683 		aRes.nStart = nCol1;
684 		aRes.nEnd = nCol2;
685 		switch( eCurTok )
686 		{
687 			case REM:
688 				aRes.eType = SB_COMMENT; break;
689 			case SYMBOL:
690 				aRes.eType = SB_SYMBOL; break;
691 			case FIXSTRING:
692 				aRes.eType = SB_STRING; break;
693 			case NUMBER:
694 				aRes.eType = SB_NUMBER; break;
695 			default:
696 				if( ( eCurTok >= FIRSTKWD && eCurTok <= LASTKWD )
697 				 || (eCurTok >= _CDECL_ ) )
698 					aRes.eType = SB_KEYWORD;
699 				else
700 					aRes.eType = SB_PUNCTUATION;
701 		}
702 		// Die Folge xxx.Keyword sollte nicht als Kwd geflagt werden
703 		if( aRes.eType == SB_KEYWORD
704 		 && ( eLastTok == DOT|| eLastTok == EXCLAM ) )
705 			aRes.eType = SB_SYMBOL;
706 		if( eCurTok != EOLN && aRes.nStart <= aRes.nEnd )
707 			rList.Insert( aRes, rList.Count() );
708 		if( aRes.eType == SB_COMMENT )
709 			break;
710 		eLastTok = eCurTok;
711 	}
712 	bUsedForHilite = sal_False;
713 }
714 
715