main.cpp

#include <iostream>
#include <vector>
#include <string>
#include <cctype>
#include <unordered_set>

enum class TokenType {
	KEYWORD,
	IDENTIFIER,
	INTEGER_LITERAL,
	FLOAT_LITERAL,
	DOUBLE_LITERAL,
	OCTAL_LITERAL,
	STRING_LITERAL,
	HEX_LITERAL,
	OPERATOR,
	PUNCTUTATOR,
	END_OF_FILE,
	UNKNOWN,
};

template<typename T>
bool ifContains(const std::vector<T>& targetArray, T inSearch) {
	for (auto i = 0; i < targetArray.size();i++) {
		if (inSearch == targetArray[i]) {
			return true;
		}
	}
	return false;
}

std::string tokenToString(TokenType token) {
	switch (token) {
	case TokenType::KEYWORD:
		return "KEYWORD";
	case TokenType::IDENTIFIER:
		return "IDENTIFIER";
	case TokenType::INTEGER_LITERAL:
		return "INTEGER_LITERAL";
	case TokenType::FLOAT_LITERAL:
		return "FLOAT_LITERAL";
	case TokenType::DOUBLE_LITERAL:
		return "DOUBLE_LITERAL";
	case TokenType::OCTAL_LITERAL:
		return "OCTAL_LITERAL";
	case TokenType::STRING_LITERAL:
		return "STRING_LITERAL";
	case TokenType::HEX_LITERAL:
		return "HEX_LITERAL";
	case TokenType::OPERATOR:
		return "OPERATOR";
	case TokenType::PUNCTUTATOR:
		return "PUNCTUTATOR";
	case TokenType::END_OF_FILE:
		return "END_OF_FILE";
	case TokenType::UNKNOWN:
		return "UNKNOWN";
	default:
		return "UNKNOWN";
	}
	
}

const std::vector<char> operators = {
'[',
']',
':',
'.',
'?',
'#',
'+',
'-',
'*',
'/',
'%',
'^',
'&',
'|',
'~',
'!',
'=',
'<',
'>'
};

const std::vector<char> punctutators = {
	'{',
	'}',
	'(',
	')',
	';',
	','
};

const std::vector<std::string> keywords = { "alignas",
"alignof",
"and",
"and_eq",
"asm",
"atomic_cancel",
"atomic_commit",
"atomic_noexcept",
"auto",
"bitand",
"bitor",
"bool",
"break",
"case",
"catch",
"char",
"char8_t",
"char16_t",
"char32_t",
"class",
"compl",
"concept",
"const",
"consteval",
"constexpr",
"constinit",
"const_cast",
"continue",
"co_await",
"co_return",
"co_yield",
"decltype",
"default",
"delete",
"do",
"double",
"dynamic_cast",
"else",
"enum",
"explicit",
"export",
"extern",
"false",
"float",
"for",
"friend",
"goto",
"if",
"inline",
"int",
"long",
"mutable",
"namespace",
"new",
"noexcept",
"not",
"not_eq",
"nullptr",
"operator",
"or",
"or_eq",
"private",
"protected",
"public",
"reflexpr",
"register",
"reinterpret_cast",
"requires",
"return",
"short",
"signed",
"sizeof",
"static",
"static_assert",
"static_cast",
"struct",
"switch",
"template",
"this",
"thread_local",
"throw",
"true",
"try",
"typedef",
"typeid",
"typename",
"union",
"unsigned",
"using",
"virtual",
"void",
"volatile",
"wchar_t",
"while",
"xor",
"xor_eq" };



class Phrase {
private:
	std::string word;
	int line = 1, column = 1;

public:
	TokenType tokenize(std::string value) {
		TokenType tokenType;
		if (ifContains(keywords,value)) {
			tokenType = TokenType::KEYWORD;
			return tokenType;
		}

		else if (ifContains(punctutators, value.at(0))) {
			tokenType = TokenType::PUNCTUTATOR;
			return tokenType;
		}

		else if (ifContains(operators, value.at(0))) {
			tokenType = TokenType::OPERATOR;
			return tokenType;
		}

		else if (value.at(0) >= '1' && value.at(0) <= '9') {
			for (int i = 0; i < value.length();i++) {
				if (!(value.at(i) == '.')) {
					tokenType = TokenType::INTEGER_LITERAL;
					return tokenType;
				}
			}
			tokenType = TokenType::FLOAT_LITERAL;
			return tokenType;
		}

		else if (value.at(0) == '0') {
			if (value.at(1) == 'x' || value.at(1) == 'X') {
				tokenType = TokenType::HEX_LITERAL;
				return tokenType;
			}
			else if (value.at(1) >= '0' || value.at(1) <= '7') {
				tokenType = TokenType::OCTAL_LITERAL;
				return tokenType;
			}
			else if (value.at(1) == '.') {
				for (int i = 2;i < value.length();i++) {
					if (value.at(i) == 'f' || value.at(i) == 'F') {
						tokenType = TokenType::FLOAT_LITERAL;
						return tokenType;
					}
					
				}
				tokenType = TokenType::DOUBLE_LITERAL;
				return tokenType;
			}
			else {
				tokenType = TokenType::UNKNOWN;
				return tokenType;
			}
			
		}

		else {
			for (int i = 0; i < value.size();i++) {
				std::vector<char> characters;
				if ((value.at(i) >= 'A' && value.at(i) <= 'Z') || (value.at(i) >= 'a' && value.at(i) <= 'z')) {
					tokenType = TokenType::IDENTIFIER;
					return tokenType;
				}
				else if (value.at(i) == '"') {
					tokenType = TokenType::STRING_LITERAL;
					return tokenType;
				}
			}

		}
		return TokenType::UNKNOWN;
	}

	void tokenSentence(std::string sentence) {
		std::string word;
		std::vector<std::string> splittedWords;
		std::vector<TokenType> tokenizedSentence;

		for (int i = 0; i < sentence.size();i++) {
			if (!(sentence.at(i) == ' ')) {
				word.push_back(sentence.at(i));
				continue;
			}
			else {
				splittedWords.push_back(word);
				word = "";
			}
		}

		if (!(word == "")) {
			splittedWords.push_back(word);
		}

		auto wordCount = splittedWords.size();

		for (int i = 0; i < wordCount;i++) {
			tokenizedSentence.push_back(tokenize(splittedWords[i]));
		}

		tokenizedSentence.push_back(TokenType::END_OF_FILE);

		for (int i = 0; i < wordCount+1;i++) {
			std::cout << tokenToString(tokenizedSentence[i]) << " ";
		}
		
	}

};

int main() {

	Phrase myLexer;
	myLexer.tokenSentence("int sayi = 5 ;");

	return 0;
}