Tokenizer.h

Go to the documentation of this file.

// Copyright (c) 2009-2014 Alan Wright. All rights reserved.
// Distributable under the terms of either the Apache License (Version 2.0)
// or the GNU Lesser General Public License.
 
#ifndef TOKENIZER_H
#define TOKENIZER_H
 
#include "TokenStream.h"
 
namespace Lucene {

class LPPAPI Tokenizer : public TokenStream {
protected:
    Tokenizer();

    Tokenizer(const ReaderPtr& input);

    Tokenizer(const AttributeFactoryPtr& factory);

    Tokenizer(const AttributeFactoryPtr& factory, const ReaderPtr& input);

    Tokenizer(const AttributeSourcePtr& source);

    Tokenizer(const AttributeSourcePtr& source, const ReaderPtr& input);
 
public:
    virtual ~Tokenizer();
 
    LUCENE_CLASS(Tokenizer);
 
protected:
    ReaderPtr input;
    CharStreamPtr charStream;
 
public:
    virtual void close();

    virtual int32_t correctOffset(int32_t currentOff);
 
    using TokenStream::reset;

    virtual void reset(const ReaderPtr& input);
};
 
}
 
#endif