Changeset 60

Timestamp:

1999-09-17 15:36:27 (9 years ago)

Author:

richard

Message:

Calculates weights for terms and documents.
Assumes 1000 docs in collection, since can't currently calculate this.

Location:

trunk/xapian-core

Files:

• backends/da/da_database.cc (modified) (8 diffs)
• backends/da/da_database.h (modified) (3 diffs)
• common/omtypes.h (modified) (1 diff)
• tests/dbtest.cc (modified) (1 diff)

trunk/xapian-core/backends/da/da_database.cc

@@ -2 +2 @@
 
 #include <string.h>
+#include <stdio.h>
 #include <errno.h>
+#include <math.h>
 #include <string>
 
@@ -9 +11 @@
 #include "daread.h"
 
-DAPostList::DAPostList(struct postings *pl, doccount tf) {
+DAPostList::DAPostList(struct postings *pl, doccount tf, doccount size) {
     termfreq = tf;
     postlist = pl;
+    termweight = log((size - tf) / tf);
+
+    printf("(dbsize, termfreq) = (%4d, %4d)\t=> termweight = %f\n",
+           size, tf, termweight);
+
     DAreadpostings(postlist, 0, 0);
 }
@@ -28 +35 @@
 }
 
+/* This is the biggie */
 weight DAPostList::get_weight() {
     if(at_end()) throw OmError("Attempt to access beyond end of postlist.");
-    return postlist->wdf;
+    doccount wdf;
+    weight wt;
+
+    wdf = postlist->wdf;
+
+    printf("(wdf, termweight)  = (%4d, %4.2f)", wdf, termweight);
+
+    double k = 1;
+    // FIXME - precalculate this freq score for several values of wt - may
+    // remove much computation.
+    wt = (double) wdf / (k + wdf);
+//    printf("(freq score %4.2f)", wt);
+
+    wt *= termweight;
+
+    printf("\t=> weight = %f\n", wt);
+
+    return wt;
 }
 
@@ -69 +94 @@
     DA_r = DAopen((byte *)(filename_r.c_str()), DARECS);
     if(DA_r == NULL)
-        throw OpeningError(string("Opening ") + filename_r + ": " + strerror(errno));
+        throw OpeningError(string("When opening ") + filename_r + ": " + strerror(errno));
 
     DA_t = DAopen((byte *)(filename_t.c_str()), DATERMS);
@@ -75 +100 @@
         DAclose(DA_r);
         DA_r = NULL;
-        throw OpeningError(string("Opening ") + filename_t + ": " + strerror(errno));
+        throw OpeningError(string("When opening ") + filename_t + ": " + strerror(errno));
     }
+
+    dbsize = 1000;  /* FIXME - read from database */
+
     opened = true;
 
@@ -117 +145 @@
     postlist = DAopenpostings(&ti, DA_t);
 
-    DAPostList * pl = new DAPostList(postlist, ti.freq);
+    DAPostList * pl = new DAPostList(postlist, ti.freq, dbsize);
     return pl;
 }
@@ -130 +158 @@
 DADatabase::term_name_to_id(termname name)
 {
+    if(!opened) throw OmError("DADatabase not opened.");
     termid id;
 
@@ -146 +175 @@
 DADatabase::term_id_to_name(termid id)
 {
+    if(!opened) throw OmError("DADatabase not opened.");
     if (id <= 0 || id > termidvec.size()) throw RangeError("invalid termid");
 //    printf("Looking up termid %d: name = `%s'\n", id, termidvec[id - 1].c_str());

trunk/xapian-core/backends/da/da_database.h

@@ -13 +13 @@
         docid  currdoc;
         doccount termfreq;
+        weight termweight;
 
-        DAPostList(struct postings *pl, doccount tf);
+        DAPostList(struct postings *pl, doccount termf, doccount dbsize);
     public:
         ~DAPostList();
@@ -21 +22 @@
 
         docid  get_docid();     // Gets current docid
-        docid  get_weight();    // Gets current weight
+        weight get_weight();    // Gets current weight
         void   next();          // Moves to next docid
         void   skip_to(docid);  // Moves to next docid >= specified docid
@@ -36 +37 @@
         struct DAfile * DA_r;
         struct DAfile * DA_t;
+        doccount dbsize;
 
         termid max_termid;

trunk/xapian-core/common/omtypes.h

@@ -9 +9 @@
 typedef docid doccount;
 
-typedef unsigned int weight;
+typedef double weight;
 
 #ifdef __cplusplus

trunk/xapian-core/tests/dbtest.cc

@@ -21 +21 @@
             did = postlist->get_docid();
             wt = postlist->get_weight();
-            printf("TermId: %d  DocId: %d  Weight: %d\n", tid, did, wt);
+            printf("TermId: %d  DocId: %d  Weight: %f\n", tid, did, wt);
             if(did == 120) postlist->skip_to(144);
             else postlist->next();