Some improvements.
--- a/Makefile Sat Aug 11 14:09:03 2007 +0200
+++ b/Makefile Sat Aug 11 15:52:30 2007 +0200
@@ -1,5 +1,17 @@
-CFLAGS=-g
+CFLAGS=-O2 -g
CC=gcc
-dictre: load.o dict.o write.o sort.o filter.o main.o
+all: dictre idx2index
+
+idx2index: idx2index.c
+
+dictre: load.o dict.o write.o sort.o filter.o main.o fastmalloc.o
$(CC) -o $@ $^
+
+dict.c: dictre.h
+write.c: dictre.h
+load.c: dictre.h
+sort.c: dictre.h
+filter.c: dictre.h
+main.c: dictre.h
+fastmalloc.c: dictre.h
--- a/dictre.h Sat Aug 11 14:09:03 2007 +0200
+++ b/dictre.h Sat Aug 11 15:52:30 2007 +0200
@@ -35,3 +35,6 @@
void load_init();
void load_dictionary(FILE *index, FILE *fdefs);
void print_words();
+
+/* fastmalloc */
+void * fastmalloc(int newsize);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/fastmalloc.c Sat Aug 11 15:52:30 2007 +0200
@@ -0,0 +1,36 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include "dictre.h"
+
+static int given = 0;
+static int sysallocated = 0;
+static void *base = 0;
+
+enum {
+ STEP = 5*1024*1024
+};
+
+void * fastmalloc(int newsize)
+{
+ void *outptr;
+
+ outptr = base + given;
+
+ given += newsize;
+
+ if (given > sysallocated)
+ {
+ if (STEP > newsize)
+ {
+ base = realloc(base, STEP);
+ sysallocated += STEP;
+ }
+ else
+ {
+ base = realloc(base, newsize);
+ sysallocated += newsize;
+ }
+ }
+
+ return outptr;
+}
--- a/filter.c Sat Aug 11 14:09:03 2007 +0200
+++ b/filter.c Sat Aug 11 15:52:30 2007 +0200
@@ -4,6 +4,7 @@
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
+#include <fcntl.h>
#include "dictre.h"
@@ -35,6 +36,7 @@
char *out;
int outsize;
int outrest;
+ int res;
out = 0;
outsize = SIZESTEP;
@@ -42,6 +44,14 @@
more_memory((void **) &out, outsize);
outrest = SIZESTEP;
+ /* We need unblocking write for select() */
+ res = fcntl(writeto, F_SETFL, O_NONBLOCK);
+ if (res == -1)
+ {
+ perror("Error setting nonblock to writeto");
+ exit(-1);
+ }
+
maxfd = writeto;
if (readfrom > maxfd)
maxfd = readfrom;
@@ -87,6 +97,19 @@
}
} while(1);
+ /*
+ {
+ int i;
+ printf("In : ");
+ for(i=0; i < deflen; ++i)
+ putchar(def[i]);
+ printf("\nOut: ");
+ for(i=0; i < outptr; ++i)
+ putchar(out[i]);
+ putchar('\n');
+ }
+ */
+
if (defptr < deflen)
{
fprintf(stderr, "Error in filter! not all written.\n");
@@ -122,7 +145,7 @@
dup(read_pipe[1]);
close(read_pipe[1]);
close(read_pipe[0]);
- execlp("bash", "bash", "-c", filter_par, 0);
+ execl(filter_par, filter_par, 0);
perror("execlp");
exit(-1);
break;
@@ -166,6 +189,8 @@
void filter_all(const char *filter_par)
{
int i;
+ static int dispndefs = 0;
+ static int filtereddefs = 0;
for(i=0; i < ndefs; ++i)
{
@@ -181,6 +206,14 @@
free(defs[i].d);
defs[i].d = newdef;
}
+ filtereddefs++; /* Not really all filtered. All but the 00-database* */
+ dispndefs++;
+ if (dispndefs >= 100)
+ {
+ dispndefs = 0;
+ printf("Filtered: %i/%i (%f)\n", filtereddefs, ndefs,
+ (float) filtereddefs / (float) ndefs);
+ }
}
}
}
--- a/load.c Sat Aug 11 14:09:03 2007 +0200
+++ b/load.c Sat Aug 11 15:52:30 2007 +0200
@@ -23,8 +23,15 @@
static void new_word(struct Word *from)
{
+ static int dispnwords = 0;
memcpy(&words[nwords], from, sizeof(*from));
nwords++;
+ dispnwords++;
+ if (dispnwords >= 1000)
+ {
+ dispnwords = 0;
+ printf("Loaded: %i\n", nwords);
+ }
}
static void new_dont_touch(int n)