WebSVN - AndroidProjects - Blame - Rev 1451 - /BauzoidTools/BauzoidNET/BauzoidNET/parser/Tokenizer.cs

Rev	Author	Line No.	Line
806	chris	1	using System;
		2	using System.Collections.Generic;
		3	using System.Linq;
		4	using System.Text;
		5	using System.Threading.Tasks;
		6
		7	namespace BauzoidNET.parser
		8	{
		9	public class Tokenizer
		10	{
1435	chris	11	public const int POSITION_STACK_SIZE = 16;
		12
806	chris	13	public const string UNEXPECTED_TOKEN = "Unexpected Token!";
		14	public const string UNEXPECTED_END_OF_STRING = "Unexpected End of String!";
		15
		16	private string mString;
		17	private int mPosition = -1;
		18	private char[] mDelimiters = {';'};
		19	private char[] mWhitespaces = {' ', '\n', '\r', '\t'};
		20	private char[] mStringDelimiters = {'"', '\''};
1435	chris	21
		22	private int[] mPositionStack = new int[POSITION_STACK_SIZE];
		23	private int mPositionStackPosition = 0;
806	chris	24
		25	public Tokenizer(string str)
		26	{
		27	mString = str;
		28	mPosition = 0;
		29	}
1435	chris	30
		31	/** Push current reading position onto stack for storage and later restore. */
		32	public void pushPosition()
		33	{
		34	if (mPositionStackPosition >= POSITION_STACK_SIZE)
		35	{
		36	return;
		37	}
		38
		39	mPositionStack[mPositionStackPosition] = mPosition;
		40	mPositionStackPosition++;
		41	}
		42
		43	/** Restore a reading position from stack. */
		44	public void popPosition()
		45	{
		46	if (mPositionStackPosition <= 0)
		47	{
		48	return;
		49	}
		50
		51	mPositionStackPosition--;
		52	mPosition = mPositionStack[mPositionStackPosition];
		53	}
806	chris	54
		55	public string readToken(string token)
		56	{
		57	skipWhitespaces();
		58
		59	if (isEndOfString())
		60	throw new ScanException(UNEXPECTED_END_OF_STRING, getSurroundings());
		61
		62	if (!isNextString(token))
		63	throw new ScanException(UNEXPECTED_TOKEN + " Expected: " + token, getSurroundings());
		64
		65	mPosition += token.Length;
		66
		67	return token;
		68	}
		69
		70	/** Read until a token or the end of the string is encountered. Does not consume the token itself. */
		71	public string readUntilToken(string token)
		72	{
		73	skipWhitespaces();
		74
		75	int startPos = mPosition;
		76
		77	string result = "";
		78
		79	if (isEndOfString())
		80	return result;
		81
		82	while (!isNextString(token) && !isEndOfString())
		83	{
		84	skipChar();
		85	}
1423	chris	86
		87	return mString.Substring(startPos, mPosition - startPos + 1);
806	chris	88	}
		89
		90	/** Read until the end of the string. Advances the string marker to the end. */
		91	public string readUntilEndOfString()
		92	{
		93	skipWhitespaces();
		94
		95	if (isEndOfString())
		96	return "";
		97
		98	String result = mString.Substring(mPosition, mString.Length);
		99	mPosition = mString.Length-1;
		100	return Preprocessor.trim(result, mWhitespaces);
		101	}
		102
		103	/** Read until the next new line character, or the end of the string. Advances the marker. */
		104	public string readUntilNewLine()
		105	{
		106	skipWhitespaces();
		107
		108	if (isEndOfString())
		109	return "";
		110
		111	int prevPos = mPosition;
		112
		113	while (!isNextString("\n") && !isEndOfString())
		114	{
		115	skipChar();
		116	}
		117
1423	chris	118	string result = mString.Substring(prevPos, mPosition-prevPos+1);
806	chris	119	return Preprocessor.trim(result, mWhitespaces);
		120	}
		121
		122	public bool checkToken(string token)
		123	{
1450	chris	124	skipWhitespaces();
		125
		126	if (isEndOfString())
		127	return false;
		128
		129	if (!isNextString(token))
		130	return false;
		131	/*
		132	// prevent exceptions in successful paths in C#/.NET
806	chris	133	int prevPosition = mPosition;
		134	try
		135	{
		136	readToken(token);
		137	}
		138	catch (ScanException ex)
		139	{
		140	return false;
		141	}
		142	finally
		143	{
		144	mPosition = prevPosition;
1450	chris	145	}*/
806	chris	146
		147	return true;
		148	}
1450	chris	149
		150	public float readNumber()
		151	{
		152	float value = 0;
		153	readNumber(ref value);
		154	return value;
		155	}
806	chris	156
1450	chris	157	public bool readNumber(ref float outValue, bool throwExceptionOnError = true)
806	chris	158	{
		159	skipWhitespaces();
1450	chris	160
		161	if (isEndOfString())
		162	{
		163	if (throwExceptionOnError)
		164	throw new ScanException(UNEXPECTED_END_OF_STRING, getSurroundings());
		165	else
		166	return false;
		167	}
806	chris	168
		169	// read numeric values or . until a delimiter or whitespace occurs
1450	chris	170	outValue = 0;
806	chris	171	float postCommaFactor = 1.0f;
		172	bool numberFound = false;
		173	bool commaFound = false;
		174
		175	float sign = 1;
		176
		177	// check for minus
		178	if (getCurrentChar() == '-')
		179	{
		180	sign = -1;
		181	skipChar();
		182	}
		183
		184	while ((!isEndOfString()) && (isNumeric(getCurrentChar()) \|\| (getCurrentChar() == '.')))
		185	{
		186	if (isEndOfString())
		187	break;
		188
		189	if (isWhitespace(getCurrentChar()))
		190	break;
		191
		192	if (isDelimiter(getCurrentChar()))
		193	break;
		194
		195	if (isNumeric(getCurrentChar()))
		196	{
		197	numberFound = true;
		198
		199	if (!commaFound)
		200	{
		201	//result = result * 10 + Character.digit(getCurrentChar(), 10);
1450	chris	202	outValue = outValue * 10 + Convert.ToInt32(getCurrentChar().ToString(), 10);
806	chris	203	}
		204	else
		205	{
		206	//result = result + Character.digit(getCurrentChar(), 10) / postCommaFactor;
1450	chris	207	outValue = outValue + Convert.ToInt32(getCurrentChar().ToString(), 10) / postCommaFactor;
806	chris	208	postCommaFactor *= 10.0f;
		209	}
		210	}
		211	else if (getCurrentChar() == '.')
		212	{
		213	// need a digit first
		214	if (!numberFound)
		215	break;
		216
		217	// check for double commas
		218	if (commaFound)
		219	break;
		220
		221	commaFound = true;
		222	postCommaFactor = 10.0f;
		223	}
		224	else
		225	{
		226	// not a number
		227	break;
		228	}
		229
		230	skipChar();
		231	}
		232
		233	if (!numberFound)
		234	{
1450	chris	235	if (isEndOfString())
		236	{
		237	if (throwExceptionOnError)
		238	throw new ScanException(UNEXPECTED_END_OF_STRING, getSurroundings());
		239	else
		240	return false;
		241	}
		242	else
		243	{
		244	if (throwExceptionOnError)
		245	throw new ScanException(UNEXPECTED_TOKEN + " Expected: Number", getSurroundings());
		246	else
		247	return false;
		248	}
806	chris	249	}
1450	chris	250
		251	outValue *= sign;
		252
		253	return true;
806	chris	254	}
		255
		256	public bool checkNumber()
		257	{
1450	chris	258	int prevPosition = mPosition;
		259
		260	float value = 0;
		261	bool result = readNumber(ref value, false);
		262
		263	mPosition = prevPosition;
		264
		265	return result;
		266
		267	/*int prevPosition = mPosition;
806	chris	268
		269	try
		270	{
		271	readNumber();
		272	}
		273	catch (ScanException ex)
		274	{
		275	return false;
		276	}
		277	finally
		278	{
		279	mPosition = prevPosition;
		280	}
		281
1450	chris	282	return true; */
806	chris	283	}
		284
1450	chris	285	public string readIdentifier(bool throwExceptionOnError = true)
806	chris	286	{
		287	// TODO: read alphanumericwithunderscore (starting with alpha or underscore) until delimiter or whitespace
		288
		289	skipWhitespaces();
1450	chris	290
		291	if (isEndOfString())
		292	{
		293	if (throwExceptionOnError)
		294	throw new ScanException(UNEXPECTED_END_OF_STRING, getSurroundings());
		295	else
		296	return null;
		297	}
806	chris	298
		299	bool idFound = false;
		300	int startChar = mPosition;
		301	int numChars = 0;
		302
		303	while (!isEndOfString() && isAlphaNumericOrUnderscore(getCurrentChar()))
		304	{
		305	idFound = true;
		306	numChars++;
		307	skipChar();
		308	}
1450	chris	309
		310	if (!idFound)
		311	{
		312	if (throwExceptionOnError)
		313	throw new ScanException(UNEXPECTED_TOKEN + " Expected: Identifier", getSurroundings());
		314	else
		315	return null;
		316	}
806	chris	317
1450	chris	318	return mString.Substring(startChar, numChars);
806	chris	319	}
		320
		321	public bool checkIdentifier()
		322	{
1450	chris	323	int prevPosition = mPosition;
		324
		325	string id = readIdentifier(false);
		326
		327	mPosition = prevPosition;
		328
		329	if (id == null)
		330	return false;
		331
		332	return true;
		333
		334	// prevent exceptions on successful path execution on .NET
		335	/*int prevPosition = mPosition;
806	chris	336
		337	try
		338	{
		339	readIdentifier();
		340	}
		341	catch (ScanException ex)
		342	{
		343	return false;
		344	}
		345	finally
		346	{
		347	mPosition = prevPosition;
1450	chris	348	}*/
806	chris	349
		350	}
		351
1450	chris	352	public string readString(bool throwExceptionOnError = true)
806	chris	353	{
		354	skipWhitespaces();
1450	chris	355
		356	if (isEndOfString())
		357	{
		358	if (throwExceptionOnError)
		359	throw new ScanException(UNEXPECTED_END_OF_STRING, getSurroundings());
		360	else
		361	return null;
		362	}
806	chris	363
		364	char usedStringDelimiter = mStringDelimiters[0];
		365	bool foundStringDelimiter = false;
		366	for (int i = 0; i < mStringDelimiters.Length; i++)
		367	{
		368	char stringDelimiter = mStringDelimiters[i];
		369	if (getCurrentChar() == stringDelimiter)
		370	{
		371	foundStringDelimiter = true;
		372	usedStringDelimiter = stringDelimiter;
		373	}
		374	}
1450	chris	375	if (!foundStringDelimiter)
		376	{
		377	if (throwExceptionOnError)
		378	throw new ScanException(UNEXPECTED_TOKEN + " Expected: String", getSurroundings());
		379	else
		380	return null;
		381	}
806	chris	382
		383	// skip first string delimiter
		384	skipChar();
		385
		386	int startChar = mPosition;
		387	int numChars = 0;
		388
		389	while (getCurrentChar() != usedStringDelimiter)
		390	{
1450	chris	391	if (isEndOfString())
		392	{
		393	if (throwExceptionOnError)
		394	throw new ScanException(UNEXPECTED_END_OF_STRING, getSurroundings());
		395	else
		396	return null;
		397	}
806	chris	398
		399	numChars++;
		400	skipChar();
		401	}
		402
		403	// skip last string delimiter
		404	skipChar();
		405
1423	chris	406	return mString.Substring(startChar, numChars);
806	chris	407	}
		408
		409	public bool checkString()
		410	{
		411	int prevPosition = mPosition;
		412
1450	chris	413	string str = readString();
		414
		415	mPosition = prevPosition;
		416
		417	if (str == null)
		418	return false;
806	chris	419
		420	return true;
		421	}
		422
		423	private bool isWhitespace(char c)
		424	{
		425	for (int i = 0; i < mWhitespaces.Length; i++)
		426	{
		427	char whitespace = mWhitespaces[i];
		428	if (c == whitespace)
		429	return true;
		430	}
		431	return false;
		432	}
		433
		434	private void skipWhitespaces()
		435	{
		436	if (isEndOfString())
		437	return;
		438
		439	while (isWhitespace(getCurrentChar()))
		440	{
		441	skipChar();
		442
		443	if (isEndOfString())
		444	return;
		445	}
		446	}
		447
		448	private bool isDelimiter(char c)
		449	{
		450	for (int i = 0; i < mDelimiters.Length; i++)
		451	{
		452	char delimiter = mDelimiters[i];
		453	if (c == delimiter)
		454	return true;
		455	}
		456	return false;
		457	}
		458
		459	private void skipChar()
		460	{
		461	if (isEndOfString())
		462	return;
		463
		464	mPosition++;
		465	}
		466
		467	public bool isNextString(string str)
		468	{
		469	return Preprocessor.isNextString(mString, mPosition, str);
		470
		471	/*for (int i = 0; i < str.length(); i++)
		472	{
		473	if (mString.length() <= (mPosition+i))
		474	return false;
		475
		476	if (str.charAt(i) != mString.charAt(mPosition + i))
		477	return false;
		478	}
		479
		480	return true;*/
		481	}
		482
		483	public bool isEndOfString()
		484	{
		485	return (mPosition >= mString.Length);
		486	}
		487
		488	public bool checkNoMoreTokens()
		489	{
		490	skipWhitespaces();
		491	return isEndOfString();
		492	}
		493
		494	public static bool isNumeric(char c)
		495	{
		496	return ((c >= '0') && (c <= '9'));
		497	}
		498
		499	public static bool isUpperCaseAlpha(char c)
		500	{
		501	return ((c >= 'A') && (c <= 'Z'));
		502	}
		503
		504	public static bool isLowerCaseAlpha(char c)
		505	{
		506	return ((c >= 'a') && (c <= 'z'));
		507	}
		508
		509	public static bool isAlpha(char c)
		510	{
		511	return (isUpperCaseAlpha(c) \|\| isLowerCaseAlpha(c));
		512	}
		513
		514	public static bool isAlphaNumeric(char c)
		515	{
		516	return (isAlpha(c) \|\| isNumeric(c));
		517	}
		518
		519	public static bool isAlphaNumericOrUnderscore(char c)
		520	{
		521	return ((c == '_') \|\| isAlphaNumeric(c));
		522	}
		523
		524	public char getCurrentChar()
		525	{
		526	return mString[mPosition];
		527	}
		528
		529	public void setWhitespaces(char[] whitespaces)
		530	{
		531	mWhitespaces = whitespaces;
		532	}
		533
		534	public void setDelimiters(char[] delimiters)
		535	{
		536	mDelimiters = delimiters;
		537	}
		538
		539	public void setStringDelimiter(char[] stringDelimiters)
		540	{
		541	mStringDelimiters = stringDelimiters;
		542	}
		543
		544	public void setPosition(int position)
		545	{
		546	mPosition = position;
		547	}
		548
		549	public int getPosition()
		550	{
		551	return mPosition;
		552	}
		553
		554	/** For debugging purposes. */
		555	public string getSurroundings()
		556	{
		557	// retrieve the line number
		558	int numLines = 1;
		559	for (int i = 0; i < mPosition; i++)
		560	{
		561	if (mString[i] == '\n')
		562	numLines++;
		563	}
		564
		565	int startIndex = Math.Max(mPosition - 10, 0);
		566	int endIndex = Math.Min(mPosition + 10, mString.Length - 1);
1423	chris	567
		568	return "Line " + numLines + " <<<" + mString.Substring(startIndex, endIndex - startIndex + 1) + ">>>";
		569	}
		570
		571	/** Skip until a token or the end of the string is encountered. Does not consume token, so the next token is the one provided. */
		572	public void skipUntilToken(String token)
		573	{
		574	skipWhitespaces();
806	chris	575
1423	chris	576	if (isEndOfString())
		577	return;
		578
		579	while (!isNextString(token) && !isEndOfString())
		580	{
		581	skipChar();
		582	}
806	chris	583	}
1423	chris	584
		585	/** Skip until a token or the end of the string is encountered, but consumes the token */
		586	public void skipUntilAfterToken(String token)
		587	{
		588	skipWhitespaces();
		589
		590	if (isEndOfString())
		591	return;
		592
		593	while (!isNextString(token) && !isEndOfString())
		594	{
		595	skipChar();
		596	}
		597
		598	readToken(token);
		599	}
806	chris	600
		601	}
		602	}

Subversion Repositories AndroidProjects

(root)/BauzoidTools/BauzoidNET/BauzoidNET/parser/Tokenizer.cs - Rev 1451