Notes for Machine Problem 3
Michael Petropoulos
You can find a good example of GetTerm() and GetTermString() in stopper.c
A description of all splay tree functions can be found in SPTREE.H
Two good examples of splay trees are in SPPLAY.C and SPEXAMP.C
INPUT FILES: files.d, docs.d
PARAMETERS: StemP, MinDocFreq, EmphProxy
OUTPUT FILES: inv-kw.d, ait-skw.d, doclen.d
#include "strlist.h"
#include "stop.h"
#include "sptree.h"
// Structure that holds splay tree node data
typedef struct
{
int totdoc;
int totfreq;
list_node *doclist;
} sp_data;
// Structure that holds document list node data
typedef struct
{
int docno;
int freq;
list_node *next;
} list_node;
// Vars for DFA, GetTerm() and GetTermString() (see stopper.c)
char term[128];
StrList stop_words;
DFA machine;
// Vars for splay tree
SPTREE Terms;
SPBLK *spnode;
sp_data *sdata;
list_node *lnode;
void main (int argc, char **argv)
{
// Construct DFA (see stopper.c)
stop_words = StrListCreate();
StrListAppendFile(stop_words, "stop.wrd");
machine = BuildDFA(stop_words);
// Initialize splay tree (see SPPLAY.C and SPEXAMP.C)
spuinit( &Terms, (SP_COMPARE_FUNC *)strcmp );
// Construct splay tree
while more files in files.d {
open current file in files.d
while more documents in docs.d with FileNo == current file in files.d {
read all attributes of the current document and keep only
what you need: MsgNo (docno), BPos, TxtPos, EPos, NLines and Proxy
call postDoc to extract keywords and to put them in the splay tree
postDoc(MsgNo, current file, BPos, EPos, TxtPos, NLines, Proxy)
}
close current file in files.d
}
// traverse splay tree obtaining terms in order
for( spnode = spfhead( &Terms ); spnode != NULL; spnode = spfnext( spnode ) ) {
get the (list_node *)((sp_data *)(spnode->data))->doclist and sort it
check the MinDocFreq
output (char *)spnode->key, (sp_data *)(spnode->data)->totdoc,
(sp_data *)(spnode->totfreq) and the sorted doclist to kw-inv.d
//put the term and the totfreq in an ordered list
put the term and the totfreq in a list
}
//output the ordered list to the ait-skw.d
sort the list using totfreq and output it to ait-skw.d
don't forget to compare totfreq to MinDocFreq
}
void postDoc(int docno, FILE *ait_file, int BPos, int EPos, intTxtPos, int NLines, char *Proxy) {
int doclength = 0;
while ( NULL != GetTermString(Proxy, machine, 128, term) ) {
InstallTerm(term, docno, &Terms, true);
doclength ++; // Use EmphProxy for this counter too
}
//read doc’s abstract from ait_file using TxtPos and NLines and put it in a temp_file
//(don’t put the whole abstract in main memory, copy line by line)
//(DON’T FORGET, GetTerm() parses the whole file)
//while ( NULL != GetTerm(temp_file, machine, 128, term) ) {
//InstallTerm(term, docno, &Terms, false);
//doclength ++;
//}
(A MORE EFFICIENT APPROACH)
read doc’s abstract from ait_file using TxtPos and EPos GetTerm(ait_file, machine, 128, term);
(do while file pointer does not pass EPos)
while ( ait_file < EPos ) {
InstallTerm(term, docno, &Terms, false);
GetTerm(ait_file, machine, 128, term) ) {
doclength ++;
output doc number and doc length to file doclen.d
}
void InstallTerm( char *term, int docno, SPTREE *Terms, bool isProxy)
{
if (StemP)
term = stem(term);
if ((spnode = splookup(term, &Terms)) == NULL) { // new term
create a new sdata and a new lnode
set sdata->totdoc and sdata->totfreq; // use EmphProxy if isProxy
set lnode->docno and lnode->freq; // use EmphProxy for this freq too
connect sdata with lnode;
insert new node in splay tree with:
spinstall( (KEYTYPE)term, (DATATYPE)sdata, &Terms); // see SPPLAY.C
}
else { // term already in tree
update totdoc and totfreq of spnode // use EmphProxy if isProxy
check if the first doc in the doclist of spnode is the same with current
if yes
update freq of the doclist’s head
else
create a new lnode and set docno and freq
add lnode to the front of the doclist of spnode
}
}