WebSVN - AndroidProjects - Blame - Rev 1671 - /BauzoidApps/crawler/core/src/com/gebauz/bauzoid/parser/Tokenizer.java

Rev	Author	Line No.	Line
1671	chris	1	package com.gebauz.bauzoid.parser;
		2
		3	import com.badlogic.gdx.Gdx;
		4	import com.gebauz.bauzoid.app.Consts;
		5
		6	public class Tokenizer
		7	{
		8	public static final int POSITION_STACK_SIZE = 16;
		9
		10	public static final String UNEXPECTED_TOKEN = "Unexpected Token!";
		11	public static final String UNEXPECTED_END_OF_STRING = "Unexpected End of String!";
		12
		13	private String mString;
		14	private int mPosition = -1;
		15	private char[] mDelimiters = {';'};
		16	private char[] mWhitespaces = {' ', '\n', '\r', '\t'};
		17	private char[] mStringDelimiters = {'"', '\''};
		18
		19	private int mPositionStack[] = new int[POSITION_STACK_SIZE];
		20	private int mPositionStackPosition = 0;
		21
		22	public Tokenizer(String str)
		23	{
		24	mString = str;
		25	mPosition = 0;
		26	}
		27
		28	/** Push current reading position onto stack for storage and later restore. */
		29	public void pushPosition()
		30	{
		31	if (mPositionStackPosition >= POSITION_STACK_SIZE)
		32	{
		33	Gdx.app.log(Consts.LOG_TAG, "Position Stack size exceeded limit!");
		34	return;
		35	}
		36
		37	mPositionStack[mPositionStackPosition] = mPosition;
		38	mPositionStackPosition++;
		39	}
		40
		41	/** Restore a reading position from stack. */
		42	public void popPosition()
		43	{
		44	if (mPositionStackPosition <= 0)
		45	{
		46	Gdx.app.log(Consts.LOG_TAG, "Underflown Position Stack Position - did you forget a Push()?");
		47	return;
		48	}
		49
		50	mPositionStackPosition--;
		51	mPosition = mPositionStack[mPositionStackPosition];
		52	}
		53
		54	public String readToken(String token) throws ScanException
		55	{
		56	skipWhitespaces();
		57
		58	if (isEndOfString())
		59	throw new ScanException(UNEXPECTED_END_OF_STRING, getSurroundings());
		60
		61	if (!isNextString(token))
		62	throw new ScanException(UNEXPECTED_TOKEN + " Expected: " + token, getSurroundings());
		63
		64	mPosition += token.length();
		65
		66	return token;
		67	}
		68
		69	/** Read until a token or the end of the string is encountered. Does not consume the token itself. */
		70	public String readUntilToken(String token)
		71	{
		72	skipWhitespaces();
		73
		74	int startPos = mPosition;
		75
		76	String result = "";
		77
		78	if (isEndOfString())
		79	return result;
		80
		81	while (!isNextString(token) && !isEndOfString())
		82	{
		83	skipChar();
		84	}
		85
		86	return mString.substring(startPos, mPosition);
		87	}
		88
		89	/** Skip until a token or the end of the string is encountered. Does not consume token, so the next token is the one provided. */
		90	public void skipUntilToken(String token) throws ScanException
		91	{
		92	skipWhitespaces();
		93
		94	if (isEndOfString())
		95	return;
		96
		97	while (!isNextString(token) && !isEndOfString())
		98	{
		99	skipChar();
		100	}
		101	}
		102
		103	/** Skip until a token or the end of the string is encountered, but consumes the token */
		104	public void skipUntilAfterToken(String token) throws ScanException
		105	{
		106	skipWhitespaces();
		107
		108	if (isEndOfString())
		109	return;
		110
		111	while (!isNextString(token) && !isEndOfString())
		112	{
		113	skipChar();
		114	}
		115
		116	readToken(token);
		117	}
		118
		119	/** Read until the end of the string. Advances the string marker to the end. */
		120	public String readUntilEndOfString()
		121	{
		122	skipWhitespaces();
		123
		124	if (isEndOfString())
		125	return "";
		126
		127	String result = mString.substring(mPosition, mString.length());
		128	mPosition = mString.length()-1;
		129	return Preprocessor.trim(result, mWhitespaces);
		130	}
		131
		132	/** Read until the next new line character, or the end of the string. Advances the marker. */
		133	public String readUntilNewLine()
		134	{
		135	skipWhitespaces();
		136
		137	if (isEndOfString())
		138	return "";
		139
		140	int prevPos = mPosition;
		141
		142	while (!isNextString("\n") && !isEndOfString())
		143	{
		144	skipChar();
		145	}
		146
		147	String result = mString.substring(prevPos, mPosition);
		148	return Preprocessor.trim(result, mWhitespaces);
		149	}
		150
		151	public boolean checkToken(String token)
		152	{
		153	int prevPosition = mPosition;
		154
		155	try
		156	{
		157	readToken(token);
		158	}
		159	catch (ScanException ex)
		160	{
		161	return false;
		162	}
		163	finally
		164	{
		165	mPosition = prevPosition;
		166	}
		167
		168	return true;
		169	}
		170
		171	public float readNumber() throws ScanException
		172	{
		173	skipWhitespaces();
		174
		175	if (isEndOfString())
		176	throw new ScanException(UNEXPECTED_END_OF_STRING, getSurroundings());
		177
		178	// read numeric values or . until a delimiter or whitespace occurs
		179	float result = 0;
		180	float postCommaFactor = 1.0f;
		181	boolean numberFound = false;
		182	boolean commaFound = false;
		183
		184	float sign = 1;
		185
		186	// check for minus
		187	if (getCurrentChar() == '-')
		188	{
		189	sign = -1;
		190	skipChar();
		191	}
		192
		193	while ((!isEndOfString()) && (isNumeric(getCurrentChar()) \|\| (getCurrentChar() == '.')))
		194	{
		195	if (isEndOfString())
		196	break;
		197
		198	if (isWhitespace(getCurrentChar()))
		199	break;
		200
		201	if (isDelimiter(getCurrentChar()))
		202	break;
		203
		204	if (isNumeric(getCurrentChar()))
		205	{
		206	numberFound = true;
		207
		208	if (!commaFound)
		209	{
		210	result = result * 10 + Character.digit(getCurrentChar(), 10);
		211	}
		212	else
		213	{
		214	result = result + Character.digit(getCurrentChar(), 10) / postCommaFactor;
		215	postCommaFactor *= 10.0f;
		216	}
		217	}
		218	else if (getCurrentChar() == '.')
		219	{
		220	// need a digit first
		221	if (!numberFound)
		222	break;
		223
		224	// check for double commas
		225	if (commaFound)
		226	break;
		227
		228	commaFound = true;
		229	postCommaFactor = 10.0f;
		230	}
		231	else
		232	{
		233	// not a number
		234	break;
		235	}
		236
		237	skipChar();
		238	}
		239
		240	if (!numberFound)
		241	{
		242	if (isEndOfString())
		243	throw new ScanException(UNEXPECTED_END_OF_STRING, getSurroundings());
		244	else
		245	throw new ScanException(UNEXPECTED_TOKEN + " Expected: Number", getSurroundings());
		246	}
		247
		248	return (sign*result);
		249	}
		250
		251	public boolean checkNumber()
		252	{
		253	int prevPosition = mPosition;
		254
		255	try
		256	{
		257	readNumber();
		258	}
		259	catch (ScanException ex)
		260	{
		261	return false;
		262	}
		263	finally
		264	{
		265	mPosition = prevPosition;
		266	}
		267
		268	return true;
		269	}
		270
		271	public String readIdentifier() throws ScanException
		272	{
		273	// TODO: read alphanumericwithunderscore (starting with alpha or underscore) until delimiter or whitespace
		274
		275	skipWhitespaces();
		276
		277	if (isEndOfString())
		278	throw new ScanException(UNEXPECTED_END_OF_STRING, getSurroundings());
		279
		280	boolean idFound = false;
		281	int startChar = mPosition;
		282	int numChars = 0;
		283
		284	while (!isEndOfString() && isAlphaNumericOrUnderscore(getCurrentChar()))
		285	{
		286	idFound = true;
		287	numChars++;
		288	skipChar();
		289	}
		290
		291	if (!idFound)
		292	throw new ScanException(UNEXPECTED_TOKEN + " Expected: Identifier", getSurroundings());
		293
		294	return mString.substring(startChar, startChar + numChars);
		295	}
		296
		297	public boolean checkIdentifier()
		298	{
		299	int prevPosition = mPosition;
		300
		301	try
		302	{
		303	readIdentifier();
		304	}
		305	catch (ScanException ex)
		306	{
		307	return false;
		308	}
		309	finally
		310	{
		311	mPosition = prevPosition;
		312	}
		313
		314	return true;
		315	}
		316
		317	public String readString() throws ScanException
		318	{
		319	skipWhitespaces();
		320
		321	if (isEndOfString())
		322	throw new ScanException(UNEXPECTED_END_OF_STRING, getSurroundings());
		323
		324	char usedStringDelimiter = mStringDelimiters[0];
		325	boolean foundStringDelimiter = false;
		326	for (char stringDelimiter : mStringDelimiters)
		327	{
		328	if (getCurrentChar() == stringDelimiter)
		329	{
		330	foundStringDelimiter = true;
		331	usedStringDelimiter = stringDelimiter;
		332	}
		333	}
		334	if (!foundStringDelimiter)
		335	throw new ScanException(UNEXPECTED_TOKEN + " Expected: String", getSurroundings());
		336
		337	// skip first string delimiter
		338	skipChar();
		339
		340	int startChar = mPosition;
		341	int numChars = 0;
		342
		343	while (getCurrentChar() != usedStringDelimiter)
		344	{
		345	if (isEndOfString())
		346	throw new ScanException(UNEXPECTED_END_OF_STRING, getSurroundings());
		347
		348	numChars++;
		349	skipChar();
		350	}
		351
		352	// skip last string delimiter
		353	skipChar();
		354
		355	return mString.substring(startChar, startChar + numChars);
		356	}
		357
		358	public boolean checkString()
		359	{
		360	int prevPosition = mPosition;
		361
		362	try
		363	{
		364	readString();
		365	}
		366	catch (ScanException ex)
		367	{
		368	return false;
		369	}
		370	finally
		371	{
		372	mPosition = prevPosition;
		373	}
		374
		375	return true;
		376	}
		377
		378	private boolean isWhitespace(char c)
		379	{
		380	for (char whitespace : mWhitespaces)
		381	{
		382	if (c == whitespace)
		383	return true;
		384	}
		385	return false;
		386	}
		387
		388	private void skipWhitespaces()
		389	{
		390	if (isEndOfString())
		391	return;
		392
		393	while (isWhitespace(getCurrentChar()))
		394	{
		395	skipChar();
		396
		397	if (isEndOfString())
		398	return;
		399	}
		400	}
		401
		402	private boolean isDelimiter(char c)
		403	{
		404	for (char delimiter : mDelimiters)
		405	{
		406	if (c == delimiter)
		407	return true;
		408	}
		409	return false;
		410	}
		411
		412	private void skipChar()
		413	{
		414	if (isEndOfString())
		415	return;
		416
		417	mPosition++;
		418	}
		419
		420	public boolean isNextString(String str)
		421	{
		422	return Preprocessor.isNextString(mString, mPosition, str);
		423
		424	/*for (int i = 0; i < str.length(); i++)
		425	{
		426	if (mString.length() <= (mPosition+i))
		427	return false;
		428
		429	if (str.charAt(i) != mString.charAt(mPosition + i))
		430	return false;
		431	}
		432
		433	return true;*/
		434	}
		435
		436	public final boolean isEndOfString()
		437	{
		438	return (mPosition >= mString.length());
		439	}
		440
		441	public final boolean checkNoMoreTokens()
		442	{
		443	skipWhitespaces();
		444	return isEndOfString();
		445	}
		446
		447	public static boolean isNumeric(char c)
		448	{
		449	return ((c >= '0') && (c <= '9'));
		450	}
		451
		452	public static boolean isUpperCaseAlpha(char c)
		453	{
		454	return ((c >= 'A') && (c <= 'Z'));
		455	}
		456
		457	public static boolean isLowerCaseAlpha(char c)
		458	{
		459	return ((c >= 'a') && (c <= 'z'));
		460	}
		461
		462	public static boolean isAlpha(char c)
		463	{
		464	return (isUpperCaseAlpha(c) \|\| isLowerCaseAlpha(c));
		465	}
		466
		467	public static boolean isAlphaNumeric(char c)
		468	{
		469	return (isAlpha(c) \|\| isNumeric(c));
		470	}
		471
		472	public static boolean isAlphaNumericOrUnderscore(char c)
		473	{
		474	return ((c == '_') \|\| isAlphaNumeric(c));
		475	}
		476
		477	public final char getCurrentChar()
		478	{
		479	return mString.charAt(mPosition);
		480	}
		481
		482	public final void setWhitespaces(char[] whitespaces)
		483	{
		484	mWhitespaces = whitespaces;
		485	}
		486
		487	public final void setDelimiters(char[] delimiters)
		488	{
		489	mDelimiters = delimiters;
		490	}
		491
		492	public final void setStringDelimiter(char[] stringDelimiters)
		493	{
		494	mStringDelimiters = stringDelimiters;
		495	}
		496
		497	public final void setPosition(int position)
		498	{
		499	mPosition = position;
		500	}
		501
		502	public final int getPosition()
		503	{
		504	return mPosition;
		505	}
		506
		507	/** For debugging purposes. */
		508	public final String getSurroundings()
		509	{
		510	// retrieve the line number
		511	int numLines = 1;
		512	for (int i = 0; i < mPosition; i++)
		513	{
		514	if (mString.charAt(i) == '\n')
		515	numLines++;
		516	}
		517
		518	int startIndex = Math.max(mPosition - 10, 0);
		519	int endIndex = Math.min(mPosition + 10, mString.length() - 1);
		520
		521	return "Line " + numLines + " <<<" + mString.substring(startIndex, endIndex) + ">>>";
		522	}
		523
		524
		525	}
		526
		527
		528

Subversion Repositories AndroidProjects

(root)/BauzoidApps/crawler/core/src/com/gebauz/bauzoid/parser/Tokenizer.java - Rev 1671