dlib C++ Library - cpp_tokenizer_kernel_1.h

// Copyright (C) 2005 Davis E. King (davis@dlib.net)
// License: Boost Software License See LICENSE.txt for the full license.
#ifndef DLIB_CPP_TOKENIZER_KERNEl_1_
#define DLIB_CPP_TOKENIZER_KERNEl_1_
#include <string>
#include <iostream>
#include "cpp_tokenizer_kernel_abstract.h"
#include "../algs.h"
namespace dlib
{
 namespace cpp_tok_kernel_1_helper
 {
 struct token_text_pair
 {
 std::string token;
 int type=0;
 }; 
 }
 template <
 typename tok,
 typename queue,
 typename set
 >
 class cpp_tokenizer_kernel_1 
 {
 /*!
 REQUIREMENTS ON tok
 tok must be an implementation of tokenizer/tokenizer_kernel_abstract.h
 REQUIREMENTS ON queue
 queue must be an implementation of queue/queue_kernel_abstract.h
 and must have T==cpp_tok_kernel_1_helper::token_text_pair
 REQUIREMENTS ON set
 set must be an implemention of set/set_kernel_abstract.h or
 hash_set/hash_set_kernel_abstract.h and must have T==std::string.
 INITIAL VALUE
 - keywords == a set of all the C++ keywords
 - tokenizer.stream_is_set() == false
 - buffer.size() == 0
 - tokenizer.get_identifier_head() == "$_" + tokenizer.lowercase_letters() + 
 tokenizer.uppercase_letters()
 - tokenizer.get_identifier_body() == "$_" + tokenizer.lowercase_letters() + 
 tokenizer.uppercase_letters() + tokenizer.numbers()
 - have_peeked == false
 CONVENTION 
 - tokenizer.stream_is_set() == stream_is_set()
 - tokenizer.get_stream() == get_stream()
 - keywords == a set of all the C++ keywords
 - tokenizer.get_identifier_head() == "$_" + tokenizer.lowercase_letters() + 
 tokenizer.uppercase_letters()
 - tokenizer.get_identifier_body() == "$_" + tokenizer.lowercase_letters() + 
 tokenizer.uppercase_letters() + tokenizer.numbers()
 - buffer == a queue of tokens. This is where we put tokens 
 we gathered early due to looking ahead.
 - if (have_peeked) then
 - next_token == the next token to be returned from get_token()
 - next_type == the type of token in peek_token
 !*/
 typedef cpp_tok_kernel_1_helper::token_text_pair token_text_pair;
 public:
 enum 
 {
 END_OF_FILE,
 KEYWORD,
 COMMENT,
 SINGLE_QUOTED_TEXT,
 DOUBLE_QUOTED_TEXT,
 IDENTIFIER,
 OTHER,
 NUMBER,
 WHITE_SPACE
 };
 cpp_tokenizer_kernel_1 ( 
 );
 virtual ~cpp_tokenizer_kernel_1 (
 );
 void clear(
 );
 void set_stream (
 std::istream& in
 );
 bool stream_is_set (
 ) const;
 std::istream& get_stream (
 ) const;
 void get_token (
 int& type,
 std::string& token
 );
 int peek_type (
 ) const;
 const std::string& peek_token (
 ) const;
 void swap (
 cpp_tokenizer_kernel_1<tok,queue,set>& item
 );
 private:
 void buffer_token(
 int type,
 const std::string& token
 )
 /*!
 ensures
 - stores the token and its type into buffer
 !*/
 {
 token_text_pair temp;
 temp.token = token;
 temp.type = type;
 buffer.enqueue(temp);
 }
 void buffer_token(
 int type,
 char token
 )
 /*!
 ensures
 - stores the token and its type into buffer
 !*/
 {
 token_text_pair temp;
 temp.token = token;
 temp.type = type;
 buffer.enqueue(temp);
 }
 // restricted functions
 cpp_tokenizer_kernel_1(const cpp_tokenizer_kernel_1<tok,queue,set>&); // copy constructor
 cpp_tokenizer_kernel_1<tok,queue,set>& operator=(const cpp_tokenizer_kernel_1<tok,queue,set>&); // assignment operator
 // data members
 set keywords;
 queue buffer;
 tok tokenizer;
 mutable std::string next_token;
 mutable int next_type;
 mutable bool have_peeked;
 }; 
 template <
 typename tok,
 typename queue,
 typename set
 >
 inline void swap (
 cpp_tokenizer_kernel_1<tok,queue,set>& a, 
 cpp_tokenizer_kernel_1<tok,queue,set>& b 
 ) { a.swap(b); } 
// ----------------------------------------------------------------------------------------
// ----------------------------------------------------------------------------------------
 // member function definitions
// ----------------------------------------------------------------------------------------
// ----------------------------------------------------------------------------------------
 template <
 typename tok,
 typename queue,
 typename set
 >
 cpp_tokenizer_kernel_1<tok,queue,set>::
 cpp_tokenizer_kernel_1( 
 ) :
 have_peeked(false)
 {
 // add C++ keywords to keywords
 std::string temp;
 temp = "#include"; keywords.add(temp);
 temp = "__asm"; keywords.add(temp);
 temp = "_asm"; keywords.add(temp); 
 temp = "if"; keywords.add(temp);
 temp = "int"; keywords.add(temp);
 temp = "else"; keywords.add(temp);
 temp = "template"; keywords.add(temp);
 temp = "void"; keywords.add(temp);
 temp = "false"; keywords.add(temp);
 temp = "class"; keywords.add(temp);
 temp = "public"; keywords.add(temp);
 temp = "while"; keywords.add(temp);
 temp = "bool"; keywords.add(temp);
 temp = "new"; keywords.add(temp);
 temp = "delete"; keywords.add(temp);
 temp = "true"; keywords.add(temp);
 temp = "typedef"; keywords.add(temp);
 temp = "const"; keywords.add(temp);
 temp = "virtual"; keywords.add(temp);
 temp = "inline"; keywords.add(temp);
 temp = "for"; keywords.add(temp);
 temp = "break"; keywords.add(temp);
 temp = "struct"; keywords.add(temp);
 temp = "float"; keywords.add(temp);
 temp = "case"; keywords.add(temp);
 temp = "enum"; keywords.add(temp);
 temp = "this"; keywords.add(temp);
 temp = "typeid"; keywords.add(temp);
 temp = "double"; keywords.add(temp);
 temp = "char"; keywords.add(temp);
 temp = "typename"; keywords.add(temp);
 temp = "signed"; keywords.add(temp);
 temp = "friend"; keywords.add(temp);
 temp = "wint_t"; keywords.add(temp);
 temp = "default"; keywords.add(temp);
 temp = "asm"; keywords.add(temp);
 temp = "reinterpret_cast"; keywords.add(temp);
 temp = "#define"; keywords.add(temp);
 temp = "do"; keywords.add(temp);
 temp = "continue"; keywords.add(temp);
 temp = "auto"; keywords.add(temp);
 temp = "unsigned"; keywords.add(temp);
 temp = "size_t"; keywords.add(temp);
 temp = "#undef"; keywords.add(temp);
 temp = "#pragma"; keywords.add(temp);
 temp = "namespace"; keywords.add(temp);
 temp = "private"; keywords.add(temp);
 temp = "#endif"; keywords.add(temp);
 temp = "catch"; keywords.add(temp);
 temp = "#else"; keywords.add(temp);
 temp = "register"; keywords.add(temp);
 temp = "volatile"; keywords.add(temp);
 temp = "const_cast"; keywords.add(temp);
 temp = "#end"; keywords.add(temp);
 temp = "mutable"; keywords.add(temp);
 temp = "static_cast"; keywords.add(temp);
 temp = "wchar_t"; keywords.add(temp);
 temp = "#if"; keywords.add(temp);
 temp = "protected"; keywords.add(temp);
 temp = "throw"; keywords.add(temp);
 temp = "using"; keywords.add(temp);
 temp = "dynamic_cast"; keywords.add(temp);
 temp = "#ifdef"; keywords.add(temp);
 temp = "return"; keywords.add(temp);
 temp = "short"; keywords.add(temp);
 temp = "#error"; keywords.add(temp);
 temp = "#line"; keywords.add(temp);
 temp = "explicit"; keywords.add(temp);
 temp = "union"; keywords.add(temp);
 temp = "#ifndef"; keywords.add(temp);
 temp = "try"; keywords.add(temp);
 temp = "sizeof"; keywords.add(temp);
 temp = "goto"; keywords.add(temp);
 temp = "long"; keywords.add(temp);
 temp = "#elif"; keywords.add(temp);
 temp = "static"; keywords.add(temp);
 temp = "operator"; keywords.add(temp); 
 temp = "switch"; keywords.add(temp);
 temp = "extern"; keywords.add(temp);
 // set the tokenizer's IDENTIFIER token for C++ identifiers
 tokenizer.set_identifier_token(
 "$_" + tokenizer.lowercase_letters() + tokenizer.uppercase_letters(),
 "$_" + tokenizer.lowercase_letters() + tokenizer.uppercase_letters() + 
 tokenizer.numbers()
 );
 }
// ----------------------------------------------------------------------------------------
 template <
 typename tok,
 typename queue,
 typename set
 >
 cpp_tokenizer_kernel_1<tok,queue,set>::
 ~cpp_tokenizer_kernel_1 (
 )
 {
 }
// ----------------------------------------------------------------------------------------
 template <
 typename tok,
 typename queue,
 typename set
 >
 void cpp_tokenizer_kernel_1<tok,queue,set>::
 clear(
 )
 {
 tokenizer.clear();
 buffer.clear();
 have_peeked = false;
 // set the tokenizer's IDENTIFIER token for C++ identifiers
 tokenizer.set_identifier_token(
 "$_" + tokenizer.lowercase_letters() + tokenizer.uppercase_letters(),
 "$_" + tokenizer.lowercase_letters() + tokenizer.uppercase_letters() + 
 tokenizer.numbers()
 );
 }
// ----------------------------------------------------------------------------------------
 template <
 typename tok,
 typename queue,
 typename set
 >
 void cpp_tokenizer_kernel_1<tok,queue,set>::
 set_stream (
 std::istream& in
 )
 {
 tokenizer.set_stream(in);
 buffer.clear();
 have_peeked = false;
 }
// ----------------------------------------------------------------------------------------
 template <
 typename tok,
 typename queue,
 typename set
 >
 bool cpp_tokenizer_kernel_1<tok,queue,set>::
 stream_is_set (
 ) const
 {
 return tokenizer.stream_is_set();
 }
// ----------------------------------------------------------------------------------------
 template <
 typename tok,
 typename queue,
 typename set
 >
 std::istream& cpp_tokenizer_kernel_1<tok,queue,set>::
 get_stream (
 ) const
 {
 return tokenizer.get_stream();
 }
// ----------------------------------------------------------------------------------------
 template <
 typename tok,
 typename queue,
 typename set
 >
 void cpp_tokenizer_kernel_1<tok,queue,set>::
 get_token (
 int& type,
 std::string& token
 )
 {
 if (!have_peeked)
 {
 if (buffer.size() > 0)
 {
 // just return what is in the buffer
 token_text_pair temp;
 buffer.dequeue(temp);
 type = temp.type;
 token = temp.token;
 return;
 }
 tokenizer.get_token(type,token);
 switch (type)
 {
 case tok::END_OF_FILE:
 {
 type = END_OF_FILE;
 } break;
 case tok::END_OF_LINE:
 case tok::WHITE_SPACE:
 {
 type = tokenizer.peek_type();
 if (type == tok::END_OF_LINE || type == tok::WHITE_SPACE)
 {
 std::string temp;
 do
 {
 tokenizer.get_token(type,temp);
 token += temp;
 type = tokenizer.peek_type();
 }while (type == tok::END_OF_LINE || type == tok::WHITE_SPACE);
 }
 type = WHITE_SPACE;
 } break;
 case tok::NUMBER:
 {
 // this could be a hex number such as 0xa33. we should check for this.
 if (tokenizer.peek_type() == tok::IDENTIFIER && token == "0" && 
 (tokenizer.peek_token()[0] == 'x' || tokenizer.peek_token()[0] == 'X'))
 {
 // this is a hex number so accumulate all the numbers and identifiers that follow
 // because they have to be part of the number
 std::string temp;
 tokenizer.get_token(type,temp);
 token = "0" + temp; 
 // get the rest of the hex number
 while (tokenizer.peek_type() == tok::IDENTIFIER ||
 tokenizer.peek_type() == tok::NUMBER
 )
 {
 tokenizer.get_token(type,temp);
 token += temp;
 }
 }
 // or this could be a floating point value or something with an 'e' or 'E' in it.
 else if ((tokenizer.peek_type() == tok::CHAR && tokenizer.peek_token()[0] == '.') ||
 (tokenizer.peek_type() == tok::IDENTIFIER && std::tolower(tokenizer.peek_token()[0]) == 'e'))
 {
 std::string temp;
 tokenizer.get_token(type,temp);
 token += temp;
 // now get the rest of the floating point value
 while (tokenizer.peek_type() == tok::IDENTIFIER ||
 tokenizer.peek_type() == tok::NUMBER
 )
 {
 tokenizer.get_token(type,temp);
 token += temp;
 }
 }
 type = NUMBER;
 } break;
 case tok::IDENTIFIER:
 {
 if (keywords.is_member(token))
 {
 type = KEYWORD;
 }
 else
 {
 type = IDENTIFIER;
 }
 } break;
 case tok::CHAR: 
 type = OTHER;
 switch (token[0])
 {
 case '#':
 {
 // this might be a preprocessor keyword so we should check the
 // next token
 if (tokenizer.peek_type() == tok::IDENTIFIER && 
 keywords.is_member('#'+tokenizer.peek_token()))
 {
 tokenizer.get_token(type,token);
 token = '#' + token;
 type = KEYWORD;
 }
 else
 {
 token = '#';
 type = OTHER;
 }
 }
 break;
 case '"':
 {
 std::string temp; 
 tokenizer.get_token(type,token);
 while (type != tok::END_OF_FILE)
 {
 // if this is the end of the quoted string
 if (type == tok::CHAR && token[0] == '"' &&
 (temp.size() == 0 || temp[temp.size()-1] != '\\' ||
 (temp.size() > 1 && temp[temp.size()-2] == '\\') ))
 {
 buffer_token(DOUBLE_QUOTED_TEXT,temp); 
 buffer_token(OTHER,"\"");
 break;
 }
 else
 {
 temp += token;
 }
 tokenizer.get_token(type,token);
 }
 type = OTHER;
 token = '"';
 } break;
 case '\'':
 {
 std::string temp; 
 tokenizer.get_token(type,token);
 if (type == tok::CHAR && token[0] == '\\')
 {
 temp += '\\';
 tokenizer.get_token(type,token);
 }
 temp += token;
 buffer_token(SINGLE_QUOTED_TEXT,temp);
 // The next character should be a ' so take it out and put it in
 // the buffer.
 tokenizer.get_token(type,token);
 buffer_token(OTHER,token);
 type = OTHER;
 token = '\'';
 } break;
 case '/':
 { 
 // look ahead to see if this is the start of a comment
 if (tokenizer.peek_type() == tok::CHAR)
 {
 if (tokenizer.peek_token()[0] == '/')
 {
 tokenizer.get_token(type,token);
 // this is the start of a line comment
 token = "//";
 std::string temp; 
 tokenizer.get_token(type,temp);
 while (type != tok::END_OF_FILE)
 {
 // if this is the end of the comment
 if (type == tok::END_OF_LINE && 
 token[token.size()-1] != '\\' )
 {
 token += '\n';
 break;
 }
 else
 {
 token += temp;
 }
 tokenizer.get_token(type,temp);
 }
 type = COMMENT;
 }
 else if (tokenizer.peek_token()[0] == '*')
 {
 tokenizer.get_token(type,token);
 // this is the start of a block comment
 token = "/*";
 std::string temp; 
 tokenizer.get_token(type,temp);
 while (type != tok::END_OF_FILE)
 {
 // if this is the end of the comment
 if (type == tok::CHAR && temp[0] == '/' &&
 token[token.size()-1] == '*')
 {
 token += '/';
 break;
 }
 else
 {
 token += temp;
 }
 tokenizer.get_token(type,temp);
 } 
 type = COMMENT; 
 }
 }
 } break;
 default:
 break;
 } // switch (token[0])
 } // switch (type)
 }
 else
 {
 // if we get this far it means we have peeked so we should 
 // return the peek data.
 type = next_type;
 token = next_token;
 have_peeked = false;
 }
 }
// ----------------------------------------------------------------------------------------
 template <
 typename tok,
 typename queue,
 typename set
 >
 int cpp_tokenizer_kernel_1<tok,queue,set>::
 peek_type (
 ) const
 {
 const_cast<cpp_tokenizer_kernel_1<tok,queue,set>*>(this)->get_token(next_type,next_token);
 have_peeked = true;
 return next_type;
 }
// ----------------------------------------------------------------------------------------
 template <
 typename tok,
 typename queue,
 typename set
 >
 const std::string& cpp_tokenizer_kernel_1<tok,queue,set>::
 peek_token (
 ) const
 {
 const_cast<cpp_tokenizer_kernel_1<tok,queue,set>*>(this)->get_token(next_type,next_token);
 have_peeked = true;
 return next_token;
 }
// ----------------------------------------------------------------------------------------
 template <
 typename tok,
 typename queue,
 typename set
 >
 void cpp_tokenizer_kernel_1<tok,queue,set>::
 swap (
 cpp_tokenizer_kernel_1& item
 )
 {
 tokenizer.swap(item.tokenizer);
 buffer.swap(item.buffer);
 }
// ----------------------------------------------------------------------------------------
}
#endif // DLIB_CPP_TOKENIZER_KERNEl_1_