Small tokenizer-related classes

git-svn-id: https://swig.svn.sourceforge.net/svnroot/swig/branches/gsoc2008-cherylfoil@10659 626c5289-ae23-0410-ae9c-e8d60b6d4f22

Source/DoxygenTranslator/Token.cpp

@@ -0,0 +1,32 @@
+#include "Token.h"
+#include <cstdlib>
+#include <iostream>
+#include <string>
+#include <list>
+using namespace std;
+
+
+Token::Token(int tType, string tString)
+{	
+	tokenType = tType;
+	tokenString = tString;
+}
+
+string Token::toString()
+{
+	if (tokenType == END_LINE){
+		return "{END OF LINE}";
+	}
+	if (tokenType == PARAGRAPH_END){
+		return "{END OF PARAGRAPH}";
+	}
+	if (tokenType == PLAINSTRING){
+		return tokenString;
+	}
+	if (tokenType == COMMAND){
+		return "{COMMAND : " + tokenString+ "}";
+	}
+	return "";
+}
+
+Token::	~Token(){}