tokenizer.hpp

/* Copyright (c) 2014-2015, The Nuria Project
 * The NuriaProject Framework is free software: you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation, either version 3 of the License,
 * or (at your option) any later version.
 * 
 * The NuriaProject Framework is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public License
 * along with The NuriaProject Framework.
 * If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef NURIA_TOKENIZER_HPP
#define NURIA_TOKENIZER_HPP

#include "essentials.hpp"
#include <QSharedData>
#include <functional>
#include <QVariant>
#include <QObject>
#include <regex>

namespace Nuria {

class TokenizerRulesPrivate;
class TokenizerPrivate;
class Tokenizer;

struct NURIA_CORE_EXPORT Token {
        Token (int tokenId = -1, int row = 0, int column = 0,
               const QVariant &value = QVariant ())
                : row (row), column (column), tokenId (tokenId), value (value)
        { }
        
        int tokenId;
        
        int row;
        
        int column;
        
        QVariant value;
        
        bool operator< (const Token &right) const;
        
};

class NURIA_CORE_EXPORT TokenizerRules {
public:
        
        enum WhitespaceMode {
                
                AutoHandleWhitespace = 0,
                
                ManualWhitespaceHandling
                
        };
        
        typedef std::function< bool(Token &, Tokenizer *) > TokenAction;
        
        TokenizerRules (WhitespaceMode mode = AutoHandleWhitespace);
        
        TokenizerRules (const TokenizerRules &other);
        
        TokenizerRules &operator= (const TokenizerRules &other);
        
        ~TokenizerRules ();
        
        void addStringToken (int tokenId, const QByteArray &terminal);
        
        void addRegexToken (int tokenId, const QByteArray &regularExpression);
        
        void addRegexToken (int tokenId, const std::regex &regularExpression);  
        
        void setTokenAction (int tokenId, TokenAction action);
        
        WhitespaceMode whitespaceMode () const;
        
        void setWhitespaceMode (WhitespaceMode mode);
        
private:
        friend class Tokenizer;
        QSharedDataPointer< TokenizerRulesPrivate > d;
};

class NURIA_CORE_EXPORT Tokenizer : public QObject {
        Q_OBJECT
public:
        
        Tokenizer (QObject *parent = nullptr);
        
        ~Tokenizer () override;
        
        TokenizerRules &defaultTokenizerRules ();
        
        void setDefaultTokenizerRules (const TokenizerRules &ruleSet);
        
        void addTokenizerRules (const QString &name, const TokenizerRules &ruleSet);
        
        TokenizerRules tokenizerRules (const QString &name) const;
        
        void removeTokenizerRules (const QString &name);
        
        void setCurrentTokenizerRules (const QString &name);
        
        const TokenizerRules &currentTokenizerRules () const;
        
        void tokenize (const QByteArray &data);
        
        QByteArray tokenizeData () const;
        
        Token nextToken ();
        
        bool atEnd () const;
        
        bool hasError () const;
        
        int errorColumn () const;
        
        int errorRow () const;
        
        int errorPosition () const;
        
        int currentColumn () const;
        
        int currentRow () const;
        
        int currentPosition () const;
        
        void setPosition (int position, int column, int row);
        
private:
        
        void advanceLocation (char c);
        void skipWhitespace ();
        bool readTokens ();
        bool readAndHandleTokens ();
        bool checkStringToken ();
        bool checkStringToken (const QByteArray &token, int tok);
        bool checkRegexToken ();
        bool checkRegexToken (const std::regex &regex, int tok);
        
        TokenizerPrivate *d_ptr;
        
};

}

NURIA_CORE_EXPORT QDebug operator<< (QDebug debug, const Nuria::Token &token);

#endif // NURIA_TOKENIZER_HPP