new file mode 100644
--- /dev/null
+++ b/lsflib/src/tokenizer.c
@@ -0,0 +1,52 @@
+/*
+ *  $Id: tokenizer.c 94 2008-04-05 01:27:30Z mbroeker $
+ * $URL: http://localhost/svn/c/lsflib/trunk/src/tokenizer.c $
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <ctype.h>
+#include <string.h>
+
+#include <lsf.h>
+
+Node *tokenizer (FILE * f, const char *delim)
+{
+
+    /*
+     * the main routine of the tokenizer
+     *
+     */
+
+    Node *aktuell, *begin;
+
+    char line[LINE_LENGTH];
+    char *token;
+    int tokens;
+
+    *line = tokens = 0;
+
+    aktuell = begin = addnode (NULL, "NULL");
+
+    while ((fgets (line, LINE_LENGTH, f)) != NULL) {
+        token = strtok (line, delim);
+        while (token) {
+            aktuell = addnode (aktuell, token);
+            token = strtok (NULL, delim);
+        }
+        *line = 0;
+    }
+
+    rewind (f);
+
+    aktuell = begin->next;
+    free (begin->data);
+    free (begin);
+
+    return aktuell;
+}