Index: include/xapian/stem.h =================================================================== --- include/xapian/stem.h (revision 10132) +++ include/xapian/stem.h (working copy) @@ -28,27 +28,78 @@ namespace Xapian { -/// Class representing a stemming algorithm. -class XAPIAN_VISIBILITY_DEFAULT Stem { +/// Base representing a stemming algorithm. +class XAPIAN_VISIBILITY_DEFAULT BaseStem : public Xapian::Internal::RefCntBase { + /// No copying allowed. + BaseStem(const BaseStem & o); + + /// No assignment allowed. + void operator=(const BaseStem & o); + + protected: + /** Destructor is protected since it should only be called by subclasses + * and RefCntPtr. + */ + virtual ~BaseStem() {} + + friend class Xapian::Internal::RefCntPtr; + public: - /// @private @internal Class representing the stemmer internals. - class Internal; - /// @private @internal Reference counted internals. - Xapian::Internal::RefCntPtr internal; + BaseStem() {} - /// Copy constructor. - Stem(const Stem & o); + /** Stem a word. + * + * @param word a word to stem. + * @return the stemmed form of the word. + */ + virtual std::string operator()(const std::string &word) const = 0; - /// Assignment. - void operator=(const Stem & o); + /// Return a string describing this object. + virtual std::string get_description() const = 0; +}; - /** Construct a Xapian::Stem object which doesn't change terms. +/// A stemming algorithm which doesn't change words at all. +class XAPIAN_VISIBILITY_DEFAULT IdentityStem : public BaseStem { + /// No copying allowed. + IdentityStem(const IdentityStem & o); + + /// No assignment allowed. + void operator=(const IdentityStem & o); + + public: + IdentityStem() {} + + /** Stem a word. * - * Equivalent to Stem("none"). + * Since this stemmer doesn't change words, this simply returns the word. + * + * @param word a word to stem. + * @return the word supplied. */ - Stem(); + std::string operator()(const std::string &word) const; - /** Construct a Xapian::Stem object for a particular language. + /// Return a string describing this object. + std::string get_description() const; +}; + +/// Class representing one of the snowball stemming algorithms. +class XAPIAN_VISIBILITY_DEFAULT SnowballStem : public BaseStem { + /// No copying allowed. + SnowballStem(const SnowballStem & o); + + /// No assignment allowed. + void operator=(const SnowballStem & o); + + public: + /// @private @internal Class representing the snowball stemmer internals. + class Internal; + + private: + /// @private @internal Snowball stemmer internals. + Internal * internal; + + public: + /** Construct a Xapian::SnowballStem object for a particular language. * * @param language Either the English name for the language * or the two letter ISO639 code. @@ -56,7 +107,6 @@ * The following language names are understood (aliases follow the * name): * - * - none - don't stem terms * - danish (da) * - dutch (nl) * - english (en) - Martin Porter's 2002 revision of his stemmer @@ -76,15 +126,15 @@ * @exception Xapian::InvalidArgumentError is thrown if * language isn't recognised. */ - explicit Stem(const std::string &language); + explicit SnowballStem(const std::string &language); /// Destructor. - ~Stem(); + ~SnowballStem(); /** Stem a word. * - * @param word a word to stem. - * @return the stem + * @param word a word to stem. + * @return the stemmed form of the word. */ std::string operator()(const std::string &word) const; @@ -104,6 +154,73 @@ static std::string get_available_languages(); }; +/// Class wrapping a reference counted stemming algorithm. +class XAPIAN_VISIBILITY_DEFAULT Stem { + public: + /// @private @internal Reference counted internals. + Xapian::Internal::RefCntPtr internal; + + /// Copy constructor. + Stem(const Stem & o) : internal(o.internal) { } + + /// Assignment. + void operator=(const Stem & o) { internal = o.internal; } + + /** Construct a Xapian::Stem object from a pointer to a BaseStem. + */ + Stem(Xapian::Internal::RefCntPtr internal_) + : internal(internal_) {} + + /** Construct a Xapian::Stem object which doesn't change terms. + */ + Stem() : internal(new Xapian::IdentityStem()) {} + + /** Construct a Xapian::Stem object for a particular language. + * + * This constructor is included for convenience, and is equivalent to + * Stem(new SnowballStem(language)) - except that a language parameter of + * "none" will produce a stemmer which doesn't remove any stems (ie, an + * IdentityStem stemmer). + * + * See Xapian::SnowballStem for details. + */ + explicit Stem(const std::string &language) + { + if (language == "none") + internal = new Xapian::IdentityStem(); + else + internal = new Xapian::SnowballStem(language); + } + + /** Stem a word. + * + * @param word a word to stem. + * @return the stemmed form of the word. + */ + std::string operator()(const std::string &word) const + { + return internal->operator()(word); + } + + /// Return a string describing this object. + std::string get_description() const + { + return "Xapian::Stem(" + internal->get_description() + ")"; + } + + /** Return a list of available languages. + * + * This is included for convenience, and is equivalent to + * SnowballStem.get_available_languages(). + * + * See Xapian::SnowballStem for details. + */ + static std::string get_available_languages() + { + return SnowballStem::get_available_languages(); + } +}; + } #endif // XAPIAN_INCLUDED_STEM_H Index: languages/Makefile.mk =================================================================== --- languages/Makefile.mk (revision 10132) +++ languages/Makefile.mk (working copy) @@ -56,10 +56,10 @@ $(CC_FOR_BUILD) -o languages/snowball -DDISABLE_JAVA `for f in $(snowball_sources) ; do test -f $$f && echo $$f || echo $(srcdir)/$$f ; done` .sbl.cc: - languages/snowball $< -o `echo $@|sed 's!\.cc$$!!'` -c++ -u -n InternalStem`echo $<|sed 's!.*/\(.\).*!\1!'|tr a-z A-Z``echo $<|sed 's!.*/.!!;s!\.sbl!!'` -p Stem::Internal + languages/snowball $< -o `echo $@|sed 's!\.cc$$!!'` -c++ -u -n InternalStem`echo $<|sed 's!.*/\(.\).*!\1!'|tr a-z A-Z``echo $<|sed 's!.*/.!!;s!\.sbl!!'` -p SnowballStem::Internal .sbl.h: - languages/snowball $< -o `echo $@|sed 's!\.h$$!!'` -c++ -u -n InternalStem`echo $<|sed 's!.*/\(.\).*!\1!'|tr a-z A-Z``echo $<|sed 's!.*/.!!;s!\.sbl!!'` -p Stem::Internal + languages/snowball $< -o `echo $@|sed 's!\.h$$!!'` -c++ -u -n InternalStem`echo $<|sed 's!.*/\(.\).*!\1!'|tr a-z A-Z``echo $<|sed 's!.*/.!!;s!\.sbl!!'` -p SnowballStem::Internal languages/allsnowballheaders.h: languages/generate-allsnowballheaders languages/Makefile.mk languages/generate-allsnowballheaders $(snowball_built_sources) Index: languages/steminternal.cc =================================================================== --- languages/steminternal.cc (revision 10132) +++ languages/steminternal.cc (working copy) @@ -131,18 +131,18 @@ namespace Xapian { -Stem::Internal::Internal() +SnowballStem::Internal::Internal() : p(create_s()), c(0), l(0), lb(0), bra(0), ket(0) { } -Stem::Internal::~Internal() +SnowballStem::Internal::~Internal() { lose_s(p); } string -Stem::Internal::operator()(const string & word) +SnowballStem::Internal::operator()(const string & word) { const symbol * s = reinterpret_cast(word.data()); replace_s(0, l, word.size(), s); @@ -156,7 +156,7 @@ /* Code for character groupings: utf8 cases */ -int Stem::Internal::get_utf8(int * slot) { +int SnowballStem::Internal::get_utf8(int * slot) { int b0, b1; int tmp = c; if (tmp >= l) return 0; @@ -171,7 +171,7 @@ * slot = (b0 & 0xF) << 12 | (b1 & 0x3F) << 6 | (p[tmp] & 0x3F); return 3; } -int Stem::Internal::get_b_utf8(int * slot) { +int SnowballStem::Internal::get_b_utf8(int * slot) { int b0, b1; int tmp = c; if (tmp <= lb) return 0; @@ -186,7 +186,7 @@ * slot = (p[tmp] & 0xF) << 12 | (b1 & 0x3F) << 6 | (b0 & 0x3F); return 3; } -int Stem::Internal::in_grouping_U(const unsigned char * s, int min, int max, int repeat) { +int SnowballStem::Internal::in_grouping_U(const unsigned char * s, int min, int max, int repeat) { do { int ch; int w = get_utf8(&ch); @@ -198,7 +198,7 @@ return 0; } -int Stem::Internal::in_grouping_b_U(const unsigned char * s, int min, int max, int repeat) { +int SnowballStem::Internal::in_grouping_b_U(const unsigned char * s, int min, int max, int repeat) { do { int ch; int w = get_b_utf8(&ch); @@ -210,7 +210,7 @@ return 0; } -int Stem::Internal::out_grouping_U(const unsigned char * s, int min, int max, int repeat) { +int SnowballStem::Internal::out_grouping_U(const unsigned char * s, int min, int max, int repeat) { do { int ch; int w = get_utf8(&ch); @@ -222,7 +222,7 @@ return 0; } -int Stem::Internal::out_grouping_b_U(const unsigned char * s, int min, int max, int repeat) { +int SnowballStem::Internal::out_grouping_b_U(const unsigned char * s, int min, int max, int repeat) { do { int ch; int w = get_b_utf8(&ch); @@ -234,21 +234,21 @@ return 0; } -int Stem::Internal::eq_s(int s_size, const symbol * s) { +int SnowballStem::Internal::eq_s(int s_size, const symbol * s) { if (l - c < s_size || memcmp(p + c, s, s_size * sizeof(symbol)) != 0) return 0; c += s_size; return 1; } -int Stem::Internal::eq_s_b(int s_size, const symbol * s) { +int SnowballStem::Internal::eq_s_b(int s_size, const symbol * s) { if (c - lb < s_size || memcmp(p + c - s_size, s, s_size * sizeof(symbol)) != 0) return 0; c -= s_size; return 1; } -int Stem::Internal::find_among(const struct among * v, int v_size, const unsigned char * fnum, const among_function * f) { +int SnowballStem::Internal::find_among(const struct among * v, int v_size, const unsigned char * fnum, const among_function * f) { int i = 0; int j = v_size; @@ -302,7 +302,7 @@ } /* find_among_b is for backwards processing. Same comments apply */ -int Stem::Internal::find_among_b(const struct among * v, int v_size, const unsigned char * fnum, const among_function * f) { +int SnowballStem::Internal::find_among_b(const struct among * v, int v_size, const unsigned char * fnum, const among_function * f) { int i = 0; int j = v_size; @@ -351,7 +351,7 @@ } int -Stem::Internal::replace_s(int c_bra, int c_ket, int s_size, const symbol * s) +SnowballStem::Internal::replace_s(int c_bra, int c_ket, int s_size, const symbol * s) { int adjustment; int len; @@ -377,7 +377,7 @@ return adjustment; } -int Stem::Internal::slice_check() { +int SnowballStem::Internal::slice_check() { Assert(p); if (bra < 0 || bra > ket || ket > l) { #if 0 @@ -389,19 +389,19 @@ return 0; } -int Stem::Internal::slice_from_s(int s_size, const symbol * s) { +int SnowballStem::Internal::slice_from_s(int s_size, const symbol * s) { if (slice_check()) return -1; replace_s(bra, ket, s_size, s); return 0; } -void Stem::Internal::insert_s(int c_bra, int c_ket, int s_size, const symbol * s) { +void SnowballStem::Internal::insert_s(int c_bra, int c_ket, int s_size, const symbol * s) { int adjustment = replace_s(c_bra, c_ket, s_size, s); if (c_bra <= bra) bra += adjustment; if (c_bra <= ket) ket += adjustment; } -symbol * Stem::Internal::slice_to(symbol * v) { +symbol * SnowballStem::Internal::slice_to(symbol * v) { if (slice_check()) return NULL; { int len = ket - bra; @@ -414,7 +414,7 @@ return v; } -symbol * Stem::Internal::assign_to(symbol * v) { +symbol * SnowballStem::Internal::assign_to(symbol * v) { int len = l; if (CAPACITY(v) < len) { v = increase_size(v, len); @@ -425,7 +425,7 @@ } #if 0 -void Stem::Internal::debug(int number, int line_count) { +void SnowballStem::Internal::debug(int number, int line_count) { int i; int limit = SIZE(p); /*if (number >= 0) printf("%3d (line %4d): '", number, line_count);*/ Index: languages/steminternal.h =================================================================== --- languages/steminternal.h (revision 10132) +++ languages/steminternal.h (working copy) @@ -41,7 +41,7 @@ #define CAPACITY(P) ((const int *)(const void *)(P))[-2] #define SET_CAPACITY(P, N) ((int *)(void *)(P))[-2] = N -typedef int (*among_function)(Xapian::Stem::Internal *); +typedef int (*among_function)(Xapian::SnowballStem::Internal *); struct among { int s_size; /* length of search string (in symbols) */ @@ -60,7 +60,7 @@ namespace Xapian { -class Stem::Internal : public Xapian::Internal::RefCntBase { +class SnowballStem::Internal { int slice_check(); protected: Index: languages/stem.cc =================================================================== --- languages/stem.cc (revision 10132) +++ languages/stem.cc (working copy) @@ -33,17 +33,22 @@ namespace Xapian { -Stem::Stem(const Stem & o) : internal(o.internal) { } +std::string +IdentityStem::operator()(const std::string &word) const +{ + return word; +} -void -Stem::operator=(const Stem & o) +std::string +IdentityStem::get_description() const { - internal = o.internal; + return "Xapian::IdentityStem"; } -Stem::Stem() : internal(0) { } -Stem::Stem(const std::string &language) : internal(0) { +SnowballStem::SnowballStem(const std::string &language) + : internal(0) +{ if (language.empty()) return; switch (language[0]) { case 'd': @@ -119,9 +124,6 @@ internal = new InternalStemNorwegian; return; } - if (language == "none") { - return; - } break; case 'p': if (language == "pt" || language == "portuguese") { @@ -163,20 +165,23 @@ throw Xapian::InvalidArgumentError("Language code " + language + " unknown"); } -Stem::~Stem() { } +SnowballStem::~SnowballStem() +{ + delete internal; +} string -Stem::operator()(const std::string &word) const +SnowballStem::operator()(const std::string &word) const { - if (!internal.get() || word.empty()) return word; + if (word.empty()) return word; return internal->operator()(word); } string -Stem::get_description() const +SnowballStem::get_description() const { - string desc = "Xapian::Stem("; - if (internal.get()) { + string desc = "Xapian::SnowballStem("; + if (internal) { desc += internal->get_description(); desc += ')'; } else { @@ -186,7 +191,7 @@ } string -Stem::get_available_languages() +SnowballStem::get_available_languages() { return LANGSTRING; } Index: languages/compiler/generator.c =================================================================== --- languages/compiler/generator.c (revision 10132) +++ languages/compiler/generator.c (working copy) @@ -1481,7 +1481,7 @@ if (q->type == t_routine && q->routine_called_from_among) { q->among_func_count = ++among_func_count; g->V[0] = q; - w(g, "static int t~V0(Xapian::Stem::Internal * this_ptr) {~N" + w(g, "static int t~V0(Xapian::SnowballStem::Internal * this_ptr) {~N" " return (static_cast(this_ptr))->~V0();~N" "}~N" "~N");