WebSVN - AndroidProjects - Blame - Rev 375 - /BauzoidApps/pingK/Pingk/src/com/gebauz/Bauzoid/parser/Tokenizer.java

Rev	Author	Line No.	Line
273	chris	1	package com.gebauz.Bauzoid.parser;
		2
		3	public class Tokenizer
		4	{
		5	public static final String UNEXPECTED_TOKEN = "Unexpected Token!";
		6	public static final String UNEXPECTED_END_OF_STRING = "Unexpected End of String!";
		7
		8	private String mString;
		9	private int mPosition = -1;
		10	private char[] mDelimiters = {';'};
		11	private char[] mWhitespaces = {' ', '\n', '\r'};
		12	private char[] mStringDelimiters = {'"', '\''};
		13
		14	public Tokenizer(String str)
		15	{
		16	mString = str;
		17	mPosition = 0;
		18	}
		19	public String readToken(String token) throws ScanException
		20	{
		21	skipWhitespaces();
		22
		23	if (isEndOfString())
		24	throw new ScanException(UNEXPECTED_END_OF_STRING, getSurroundings());
		25
		26	if (!isNextString(token))
		27	throw new ScanException(UNEXPECTED_TOKEN, getSurroundings());
		28
		29	mPosition += token.length();
		30
		31	return token;
		32	}
		33
		34	public boolean checkToken(String token)
		35	{
		36	int prevPosition = mPosition;
		37
		38	try
		39	{
		40	readToken(token);
		41	}
		42	catch (ScanException ex)
		43	{
		44	return false;
		45	}
		46	finally
		47	{
		48	mPosition = prevPosition;
		49	}
		50
		51	return true;
		52	}
		53
		54	public float readNumber() throws ScanException
		55	{
		56	skipWhitespaces();
		57
		58	if (isEndOfString())
		59	throw new ScanException(UNEXPECTED_END_OF_STRING, getSurroundings());
		60
		61	// read numeric values or . until a delimiter or whitespace occurs
		62	float result = 0;
		63	float postCommaFactor = 1.0f;
		64	boolean numberFound = false;
		65	boolean commaFound = false;
375	chris	66
		67	float sign = 1;
		68
		69	// check for minus
		70	if (getCurrentChar() == '-')
		71	{
		72	sign = -1;
		73	skipChar();
		74	}
273	chris	75
		76	while (isNumeric(getCurrentChar()) \|\| (getCurrentChar() == '.'))
		77	{
		78	if (isEndOfString())
		79	break;
		80
		81	if (isWhitespace(getCurrentChar()))
		82	break;
		83
		84	if (isDelimiter(getCurrentChar()))
		85	break;
		86
		87	if (isNumeric(getCurrentChar()))
		88	{
		89	numberFound = true;
		90
		91	if (!commaFound)
		92	{
		93	result = result * 10 + Character.digit(getCurrentChar(), 10);
		94	}
		95	else
		96	{
		97	result = result + Character.digit(getCurrentChar(), 10) / postCommaFactor;
		98	postCommaFactor *= 10.0f;
		99	}
		100	}
		101	else if (getCurrentChar() == '.')
		102	{
		103	// need a digit first
		104	if (!numberFound)
		105	break;
		106
		107	// check for double commas
		108	if (commaFound)
		109	break;
		110
		111	commaFound = true;
		112	postCommaFactor = 10.0f;
		113	}
		114	else
		115	{
		116	// not a number
		117	break;
		118	}
		119
		120	skipChar();
		121	}
		122
		123	if (!numberFound)
		124	{
		125	if (isEndOfString())
		126	throw new ScanException(UNEXPECTED_END_OF_STRING, getSurroundings());
		127	else
		128	throw new ScanException(UNEXPECTED_TOKEN, getSurroundings());
		129	}
		130
375	chris	131	return (sign * result);
273	chris	132	}
		133
		134	public boolean checkNumber()
		135	{
		136	int prevPosition = mPosition;
		137
		138	try
		139	{
		140	readNumber();
		141	}
		142	catch (ScanException ex)
		143	{
		144	return false;
		145	}
		146	finally
		147	{
		148	mPosition = prevPosition;
		149	}
		150
		151	return true;
		152	}
		153
		154	public String readIdentifier() throws ScanException
		155	{
		156	// TODO: read alphanumericwithunderscore (starting with alpha or underscore) until delimiter or whitespace
		157
		158	skipWhitespaces();
		159
		160	if (isEndOfString())
		161	throw new ScanException(UNEXPECTED_END_OF_STRING, getSurroundings());
		162
		163	boolean idFound = false;
		164	int startChar = mPosition;
		165	int numChars = 0;
		166
		167	while (isAlphaNumericOrUnderscore(getCurrentChar()))
		168	{
		169	idFound = true;
		170	numChars++;
		171	skipChar();
		172	}
		173
		174	if (!idFound)
		175	throw new ScanException(UNEXPECTED_TOKEN, getSurroundings());
		176
		177	return mString.substring(startChar, startChar + numChars);
		178	}
		179
		180	public boolean checkIdentifier()
		181	{
		182	int prevPosition = mPosition;
		183
		184	try
		185	{
		186	readIdentifier();
		187	}
		188	catch (ScanException ex)
		189	{
		190	return false;
		191	}
		192	finally
		193	{
		194	mPosition = prevPosition;
		195	}
		196
		197	return true;
		198	}
		199
		200	public String readString() throws ScanException
		201	{
		202	skipWhitespaces();
		203
		204	if (isEndOfString())
		205	throw new ScanException(UNEXPECTED_END_OF_STRING, getSurroundings());
		206
		207	char usedStringDelimiter = mStringDelimiters[0];
		208	boolean foundStringDelimiter = false;
		209	for (char stringDelimiter : mStringDelimiters)
		210	{
		211	if (getCurrentChar() == stringDelimiter)
		212	{
		213	foundStringDelimiter = true;
		214	usedStringDelimiter = stringDelimiter;
		215	}
		216	}
		217	if (!foundStringDelimiter)
		218	throw new ScanException(UNEXPECTED_TOKEN, getSurroundings());
		219
		220	// skip first string delimiter
		221	skipChar();
		222
		223	int startChar = mPosition;
		224	int numChars = 0;
		225
		226	while (getCurrentChar() != usedStringDelimiter)
		227	{
		228	if (isEndOfString())
		229	throw new ScanException(UNEXPECTED_END_OF_STRING, getSurroundings());
		230
		231	numChars++;
		232	skipChar();
		233	}
		234
		235	// skip last string delimiter
		236	skipChar();
		237
		238	return mString.substring(startChar, startChar + numChars);
		239	}
		240
		241	public boolean checkString()
		242	{
		243	int prevPosition = mPosition;
		244
		245	try
		246	{
		247	readString();
		248	}
		249	catch (ScanException ex)
		250	{
		251	return false;
		252	}
		253	finally
		254	{
		255	mPosition = prevPosition;
		256	}
		257
		258	return true;
		259	}
		260
		261	private boolean isWhitespace(char c)
		262	{
		263	for (char whitespace : mWhitespaces)
		264	{
		265	if (c == whitespace)
		266	return true;
		267	}
		268	return false;
		269	}
		270
		271	private void skipWhitespaces()
		272	{
		273	while (isWhitespace(getCurrentChar()))
		274	{
		275	skipChar();
		276
		277	if (isEndOfString())
		278	return;
		279	}
		280	}
		281
		282	private boolean isDelimiter(char c)
		283	{
		284	for (char delimiter : mDelimiters)
		285	{
		286	if (c == delimiter)
		287	return true;
		288	}
		289	return false;
		290	}
		291
		292	private void skipChar()
		293	{
		294	mPosition++;
		295	}
		296
		297	public boolean isNextString(String str)
		298	{
		299	for (int i = 0; i < str.length(); i++)
		300	{
		301	if (str.charAt(i) != mString.charAt(mPosition + i))
		302	return false;
		303	}
		304
		305	return true;
		306	}
		307
		308	public final boolean isEndOfString()
		309	{
		310	return (mPosition >= mString.length());
		311	}
		312
		313	public final boolean checkNoMoreTokens()
		314	{
		315	skipWhitespaces();
		316	return isEndOfString();
		317	}
		318
		319	public static boolean isNumeric(char c)
		320	{
		321	return ((c >= '0') && (c <= '9'));
		322	}
		323
		324	public static boolean isUpperCaseAlpha(char c)
		325	{
		326	return ((c >= 'A') && (c <= 'Z'));
		327	}
		328
		329	public static boolean isLowerCaseAlpha(char c)
		330	{
		331	return ((c >= 'a') && (c <= 'z'));
		332	}
		333
		334	public static boolean isAlpha(char c)
		335	{
		336	return (isUpperCaseAlpha(c) \|\| isLowerCaseAlpha(c));
		337	}
		338
		339	public static boolean isAlphaNumeric(char c)
		340	{
		341	return (isAlpha(c) \|\| isNumeric(c));
		342	}
		343
		344	public static boolean isAlphaNumericOrUnderscore(char c)
		345	{
		346	return ((c == '_') \|\| isAlphaNumeric(c));
		347	}
		348
		349	public final char getCurrentChar()
		350	{
		351	return mString.charAt(mPosition);
		352	}
		353
		354	public final void setWhitespaces(char[] whitespaces)
		355	{
		356	mWhitespaces = whitespaces;
		357	}
		358
		359	public final void setDelimiters(char[] delimiters)
		360	{
		361	mDelimiters = delimiters;
		362	}
		363
		364	public final void setStringDelimiter(char[] stringDelimiters)
		365	{
		366	mStringDelimiters = stringDelimiters;
		367	}
		368
		369	public final void setPosition(int position)
		370	{
		371	mPosition = position;
		372	}
		373
		374	public final int getPosition()
		375	{
		376	return mPosition;
		377	}
		378
		379	/** For debugging purposes. */
		380	public final String getSurroundings()
		381	{
		382	int startIndex = Math.max(mPosition - 10, 0);
		383	int endIndex = Math.min(mPosition + 10, mString.length() - 1);
		384
		385	return mString.substring(startIndex, endIndex);
		386	}
		387	}
		388
		389
		390

Subversion Repositories AndroidProjects

(root)/BauzoidApps/pingK/Pingk/src/com/gebauz/Bauzoid/parser/Tokenizer.java - Rev 375