diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | Makefile | 8 | ||||
| -rw-r--r-- | notmuch-index-message.cc | 842 | ||||
| -rw-r--r-- | xapian-dump.cc | 204 |
4 files changed, 1 insertions, 1055 deletions
@@ -1,4 +1,2 @@ -xapian-dump -notmuch-index-message notmuch @@ -1,4 +1,4 @@ -PROGS=notmuch notmuch-index-message xapian-dump +PROGS=notmuch MYCFLAGS=-Wall -O0 -g `pkg-config --cflags glib-2.0` MYCXXFLAGS=$(MYCFLAGS) `xapian-config --cxxflags` @@ -16,11 +16,5 @@ all: $(PROGS) notmuch: notmuch.o database.o date.o message.o xutil.o $(CC) $(MYLDFLAGS) $^ -o $@ -notmuch-index-message: notmuch-index-message.cc - $(CXX) $(CXXFLAGS) $(MYCXXFLAGS) notmuch-index-message.cc `pkg-config --cflags --libs gmime-2.4` `xapian-config --cxxflags --libs` -o notmuch-index-message - -xapian-dump: xapian-dump.cc - $(CXX) $(CXXFLAGS) $(MYCXXFLAGS) xapian-dump.cc `xapian-config --libs --cxxflags` -o xapian-dump - clean: rm -f $(PROGS) *.o diff --git a/notmuch-index-message.cc b/notmuch-index-message.cc deleted file mode 100644 index e3de80f1..00000000 --- a/notmuch-index-message.cc +++ /dev/null @@ -1,842 +0,0 @@ -/* - * Copyright © 2009 Carl Worth - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see http://www.gnu.org/licenses/ . - * - * Author: Carl Worth <cworth@cworth.org> - */ - -/* This indexer creates a Xapian mail index that is remarkably similar - * to that created by sup. The big difference, (and the thing that - * will keep a notmuch index from being used by sup directly), is that - * sup expects a serialized ruby data structure in the document's data - * field, but notmuch just puts the mail's filename there (trusting - * that the email client can get the data in needs from the filename). - * - * Note: One bug here is that sup actually merges together fields such - * as To, CC, Bcc etc. when finding multiple emails with the same - * message ID. To support something similar, notmuch should list - * multiple files in the data field. - * - * Other differences between sup and notmuch-index identified so far: - * - * o sup supports encrypted mime parts by prompting for a passphrase - * to decrypt the message. So far, notmuch doesn't support this, - * both because I'm lazy to code it, and I also think doing so - * would present a security leak. - * - * o sup and notmuch have different heuristics for identifying (and - * thus ignoring) signatures. For example, sup considers a line - * consisting of two hypens as a signature separator, while - * notmuch expects those two hyphens to be followed by a space - * character. - * - * o sup as been seen to split some numbers before indexing - * them. For example, the number 1754 in an email message was - * indexed by sup as separate terms 17 and 54. I couldn't find any - * explanation for this behavior and did not try to replicate it - * in notmuch. - */ - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <errno.h> -#include <time.h> -#include <sys/time.h> - -#include <iostream> - -#include <gmime/gmime.h> - -#include <xapian.h> - -using namespace std; - -#define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0])) - -/* Xapian complains if we provide a term longer than this. */ -#define NOTMUCH_MAX_TERM 245 - -/* These prefix values are specifically chosen to be compatible - * with sup, (http://sup.rubyforge.org), written by - * William Morgan <wmorgan-sup@masanjin.net>, and released - * under the GNU GPL v2. - */ - -typedef struct { - const char *name; - const char *prefix; -} prefix_t; - -prefix_t NORMAL_PREFIX[] = { - { "subject", "S" }, - { "body", "B" }, - { "from_name", "FN" }, - { "to_name", "TN" }, - { "name", "N" }, - { "attachment", "A" } -}; - -prefix_t BOOLEAN_PREFIX[] = { - { "type", "K" }, - { "from_email", "FE" }, - { "to_email", "TE" }, - { "email", "E" }, - { "date", "D" }, - { "label", "L" }, - { "source_id", "I" }, - { "attachment_extension", "O" }, - { "msgid", "Q" }, - { "thread", "H" }, - { "ref", "R" } -}; - -/* Similarly, these value numbers are also chosen to be sup - * compatible. */ - -typedef enum { - NOTMUCH_VALUE_MESSAGE_ID = 0, - NOTMUCH_VALUE_THREAD = 1, - NOTMUCH_VALUE_DATE = 2 -} notmuch_value_t; - -static const char * -find_prefix (const char *name) -{ - unsigned int i; - - for (i = 0; i < ARRAY_SIZE (NORMAL_PREFIX); i++) - if (strcmp (name, NORMAL_PREFIX[i].name) == 0) - return NORMAL_PREFIX[i].prefix; - - for (i = 0; i < ARRAY_SIZE (BOOLEAN_PREFIX); i++) - if (strcmp (name, BOOLEAN_PREFIX[i].name) == 0) - return BOOLEAN_PREFIX[i].prefix; - - return ""; -} - -/* "128 bits of thread-id ought to be enough for anybody" */ -#define NOTMUCH_THREAD_ID_BITS 128 -#define NOTMUCH_THREAD_ID_DIGITS (NOTMUCH_THREAD_ID_BITS / 4) -typedef struct _thread_id { - char str[NOTMUCH_THREAD_ID_DIGITS + 1]; -} thread_id_t; - -static void -thread_id_generate (thread_id_t *thread_id) -{ - FILE *urandom; - uint32_t value; - char *s; - int i; - - urandom = fopen ("/dev/urandom", "r"); - if (urandom == NULL) { - fprintf (stderr, "Error opening /dev/urandom: %s\n", - strerror (errno)); - fprintf (stderr, "Perhaps notmuch needs some portability fixes for your platform?\n"); - exit (1); - } - - s = thread_id->str; - for (i = 0; i < NOTMUCH_THREAD_ID_DIGITS; i += 8) { - fread ((void *) &value, sizeof (value), 1, urandom); - sprintf (s, "%08x", value); - s += 8; - } - - fclose (urandom); - - printf ("Generated thread id: %s\n", thread_id->str); -} - -static void -add_term (Xapian::Document doc, - const char *prefix_name, - const char *value) -{ - const char *prefix; - char *term; - - if (value == NULL) - return; - - prefix = find_prefix (prefix_name); - - term = g_strdup_printf ("%s%s", prefix, value); - - if (strlen (term) <= NOTMUCH_MAX_TERM) - doc.add_term (term); - - g_free (term); -} - -static void -gen_terms (Xapian::TermGenerator term_gen, - const char *prefix_name, - const char *text) -{ - const char *prefix; - - if (text == NULL) - return; - - prefix = find_prefix (prefix_name); - - term_gen.index_text (text, 1, prefix); -} - -static void -gen_terms_address_name (Xapian::TermGenerator term_gen, - InternetAddress *address, - const char *prefix_name) -{ - if (INTERNET_ADDRESS_IS_MAILBOX(address)) { - const char *name; - int own_name = 0; - - name = internet_address_get_name (address); - - /* In the absence of a name, we'll strip the part before the @ - * from the address. */ - if (! name) { - InternetAddressMailbox *mailbox = INTERNET_ADDRESS_MAILBOX (address); - const char *addr = internet_address_mailbox_get_addr (mailbox); - const char *at; - - at = strchr (addr, '@'); - if (at) { - name = strndup (addr, at - addr); - own_name = 1; - } - } - - if (name) - gen_terms (term_gen, prefix_name, name); - - if (own_name) - free ((void *) name); - } else if (INTERNET_ADDRESS_IS_GROUP (address)) { - InternetAddressGroup *group = INTERNET_ADDRESS_GROUP (address); - InternetAddressList *list = internet_address_group_get_members(group); - if (list) { - int length = internet_address_list_length(list); - int i; - - for (i = 0; i < length; i++) - gen_terms_address_name(term_gen, - internet_address_list_get_address(list, i), - prefix_name); - } - } -} - -static void -gen_terms_address_names (Xapian::TermGenerator term_gen, - InternetAddressList *addresses, - const char *address_type) -{ - int i; - InternetAddress *address; - - if (addresses == NULL) - return; - - for (i = 0; i < internet_address_list_length (addresses); i++) { - address = internet_address_list_get_address (addresses, i); - gen_terms_address_name (term_gen, address, address_type); - gen_terms_address_name (term_gen, address, "name"); - gen_terms_address_name (term_gen, address, "body"); - } -} - -static void -add_term_address_addr (Xapian::Document doc, - InternetAddress *address, - const char *prefix_name) -{ - if (INTERNET_ADDRESS_IS_MAILBOX(address)) { - InternetAddressMailbox *mailbox = INTERNET_ADDRESS_MAILBOX (address); - const char *addr; - - addr = internet_address_mailbox_get_addr (mailbox); - - if (addr) - add_term (doc, prefix_name, addr); - } else if (INTERNET_ADDRESS_IS_GROUP (address)) { - InternetAddressGroup *group = INTERNET_ADDRESS_GROUP (address); - InternetAddressList *list = internet_address_group_get_members(group); - if (list) { - int length = internet_address_list_length(list); - int i; - - for (i = 0; i < length; i++) - add_term_address_addr(doc, - internet_address_list_get_address(list, i), - prefix_name); - } - } -} - -static void -add_terms_address_addrs (Xapian::Document doc, - InternetAddressList *addresses, - const char *address_type) -{ - int i; - InternetAddress *address; - - if (addresses == NULL) - return; - - for (i = 0; i < internet_address_list_length (addresses); i++) { - address = internet_address_list_get_address (addresses, i); - add_term_address_addr (doc, address, address_type); - add_term_address_addr (doc, address, "email"); - } -} - -static const char * -skip_re_in_subject (const char *subject) -{ - const char *s = subject; - - if (subject == NULL) - return NULL; - - while (*s) { - while (*s && isspace (*s)) - s++; - if (strncasecmp (s, "re:", 3) == 0) - s += 3; - else - break; - } - - return s; -} - -static void -find_messages_by_term (Xapian::Database db, - const char *prefix_name, - const char *value, - Xapian::PostingIterator *begin, - Xapian::PostingIterator *end) -{ - Xapian::PostingIterator i; - char *term; - - term = g_strdup_printf ("%s%s", find_prefix (prefix_name), value); - - *begin = db.postlist_begin (term); - - if (end) - *end = db.postlist_end (term); - - free (term); -} - -Xapian::Document -find_message_by_docid (Xapian::Database db, Xapian::docid docid) -{ - return db.get_document (docid); -} - -Xapian::Document -find_message_by_message_id (Xapian::Database db, const char *message_id) -{ - Xapian::PostingIterator i, end; - - find_messages_by_term (db, "msgid", message_id, &i, &end); - - if (i != end) - return find_message_by_docid (db, *i); - else - return Xapian::Document (); -} - -static void -insert_thread_id (GHashTable *thread_ids, Xapian::Document doc) -{ - string value_string; - const char *value, *id, *comma; - - value_string = doc.get_value (NOTMUCH_VALUE_THREAD); - value = value_string.c_str(); - if (strlen (value)) { - id = value; - while (*id) { - comma = strchr (id, ','); - if (comma == NULL) - comma = id + strlen (id); - g_hash_table_insert (thread_ids, - strndup (id, comma - id), NULL); - id = comma; - if (*id) - id++; - } - } -} - -/* Return one or more thread_ids, (as a GPtrArray of strings), for the - * given message based on looking into the database for any messages - * referenced in parents, and also for any messages in the database - * referencing message_id. - * - * Caller should free all strings in the array and the array itself, - * (g_ptr_array_free) when done. */ -static GPtrArray * -find_thread_ids (Xapian::Database db, - GPtrArray *parents, - const char *message_id) -{ - Xapian::PostingIterator child, children_end; - Xapian::Document doc; - GHashTable *thread_ids; - GList *keys, *l; - unsigned int i; - const char *parent_message_id; - GPtrArray *result; - - thread_ids = g_hash_table_new_full (g_str_hash, g_str_equal, - free, NULL); - - find_messages_by_term (db, "ref", message_id, &child, &children_end); - for ( ; child != children_end; child++) { - doc = find_message_by_docid (db, *child); - insert_thread_id (thread_ids, doc); - } - - for (i = 0; i < parents->len; i++) { - parent_message_id = (char *) g_ptr_array_index (parents, i); - doc = find_message_by_message_id (db, parent_message_id); - insert_thread_id (thread_ids, doc); - } - - result = g_ptr_array_new (); - - keys = g_hash_table_get_keys (thread_ids); - for (l = keys; l; l = l->next) { - char *id = (char *) l->data; - g_ptr_array_add (result, id); - } - g_list_free (keys); - - /* We're done with the hash table, but we've taken the pointers to - * the allocated strings and put them into our result array, so - * tell the hash not to free them on its way out. */ - g_hash_table_steal_all (thread_ids); - g_hash_table_unref (thread_ids); - - return result; -} - -/* Add a term for each message-id in the References header of the - * message. */ -static void -parse_references (GPtrArray *array, - const char *refs_str) -{ - GMimeReferences *refs, *r; - const char *message_id; - - if (refs_str == NULL) - return; - - refs = g_mime_references_decode (refs_str); - - for (r = refs; r; r = r->next) { - message_id = g_mime_references_get_message_id (r); - g_ptr_array_add (array, g_strdup (message_id)); - } - - g_mime_references_free (refs); -} - -/* Given a string representing the body of a message, generate terms - * for it, (skipping quoted portions and signatures). */ -static void -gen_terms_body_str (Xapian::TermGenerator term_gen, - char *body) -{ - char *line, *line_end, *next_line; - - if (body == NULL) - return; - - next_line = body; - - while (1) { - line = next_line; - if (*line == '\0') - break; - - next_line = strchr (line, '\n'); - if (next_line == NULL) { - next_line = line + strlen (line); - } - line_end = next_line - 1; - - /* Get to the next non-blank line. */ - while (*next_line == '\n') - next_line++; - - /* Skip blank lines. */ - if (line_end < line) - continue; - - /* Skip lines that are quotes. */ - if (*line == '>') - continue; - - /* Also skip lines introducing a quote on the next line. */ - if (*line_end == ':' && *next_line == '>') - continue; - - /* Finally, bail as soon as we see a signature. */ - /* XXX: Should only do this if "near" the end of the message. */ - if (strncmp (line, "-- ", 3) == 0 || - strncmp (line, "----------", 10) == 0 || - strncmp (line, "__________", 10) == 0) - break; - - *(line_end + 1) = '\0'; - gen_terms (term_gen, "body", line); - } -} - - -/* Callback to generate terms for each mime part of a message. */ -static void -gen_terms_part (Xapian::TermGenerator term_gen, - GMimeObject *part) -{ - GMimeStream *stream; - GMimeDataWrapper *wrapper; - GByteArray *byte_array; - GMimeContentDisposition *disposition; - char *body; - - if (GMIME_IS_MULTIPART (part)) { - GMimeMultipart *multipart = GMIME_MULTIPART (part); - int i; - - for (i = 0; i < g_mime_multipart_get_count (multipart); i++) { - if (GMIME_IS_MULTIPART_SIGNED (multipart)) { - /* Don't index the signature. */ - if (i == 1) - continue; - if (i > 1) - fprintf (stderr, "Warning: Unexpected extra parts of mutlipart/signed. Indexing anyway.\n"); - } - gen_terms_part (term_gen, - g_mime_multipart_get_part (multipart, i)); - } - return; - } - - if (GMIME_IS_MESSAGE_PART (part)) { - GMimeMessage *message; - - message = g_mime_message_part_get_message (GMIME_MESSAGE_PART (part)); - - gen_terms_part (term_gen, g_mime_message_get_mime_part (message)); - - return; - } - - if (! (GMIME_IS_PART (part))) { - fprintf (stderr, "Warning: Not indexing unknown mime part: %s.\n", - g_type_name (G_OBJECT_TYPE (part))); - return; - } - - disposition = g_mime_object_get_content_disposition (part); - if (disposition && - strcmp (disposition->disposition, GMIME_DISPOSITION_ATTACHMENT) == 0) - { - const char *filename = g_mime_part_get_filename (GMIME_PART (part)); - const char *extension; - - add_term (term_gen.get_document (), "label", "attachment"); - gen_terms (term_gen, "attachment", filename); - - if (filename) { - extension = strchr (filename, '.'); - if (extension) { - add_term (term_gen.get_document (), "attachment_extension", - extension + 1); - } - } - - return; - } - - byte_array = g_byte_array_new (); - - stream = g_mime_stream_mem_new_with_byte_array (byte_array); - g_mime_stream_mem_set_owner (GMIME_STREAM_MEM (stream), FALSE); - wrapper = g_mime_part_get_content_object (GMIME_PART (part)); - if (wrapper) - g_mime_data_wrapper_write_to_stream (wrapper, stream); - - g_object_unref (stream); - - g_byte_array_append (byte_array, (guint8 *) "\0", 1); - body = (char *) g_byte_array_free (byte_array, FALSE); - - gen_terms_body_str (term_gen, body); - - free (body); -} - -static void -index_file (Xapian::WritableDatabase db, - Xapian::TermGenerator term_gen, - const char *filename) -{ - Xapian::Document doc; - - GMimeStream *stream; - GMimeParser *parser; - GMimeMessage *message; - InternetAddressList *addresses; - GPtrArray *parents, *thread_ids; - - FILE *file; - - const char *subject, *refs, *in_reply_to, *from; - const char *message_id; - - time_t time; - struct tm gm_time_tm; - char date_str[16]; /* YYYYMMDDHHMMSS + 1 for Y100k compatibility ;-) */ - unsigned int i; - - file = fopen (filename, "r"); - if (! file) { - fprintf (stderr, "Error opening %s: %s\n", filename, strerror (errno)); - exit (1); - } - - stream = g_mime_stream_file_new (file); - - parser = g_mime_parser_new_with_stream (stream); - - message = g_mime_parser_construct_message (parser); - - doc = Xapian::Document (); - - doc.set_data (filename); - - term_gen.set_stemmer (Xapian::Stem ("english")); - - term_gen.set_document (doc); - - from = g_mime_message_get_sender (message); - addresses = internet_address_list_parse_string (from); - - gen_terms_address_names (term_gen, addresses, "from_name"); - - addresses = g_mime_message_get_all_recipients (message); - gen_terms_address_names (term_gen, addresses, "to_name"); - - subject = g_mime_message_get_subject (message); - subject = skip_re_in_subject (subject); - gen_terms (term_gen, "subject", subject); - gen_terms (term_gen, "body", subject); - - gen_terms_part (term_gen, g_mime_message_get_mime_part (message)); - - parents = g_ptr_array_new (); - - refs = g_mime_object_get_header (GMIME_OBJECT (message), "references"); - parse_references (parents, refs); - - in_reply_to = g_mime_object_get_header (GMIME_OBJECT (message), - "in-reply-to"); - parse_references (parents, in_reply_to); - - for (i = 0; i < parents->len; i++) - add_term (doc, "ref", (char *) g_ptr_array_index (parents, i)); - - message_id = g_mime_message_get_message_id (message); - - thread_ids = find_thread_ids (db, parents, message_id); - - for (i = 0; i < parents->len; i++) - g_free (g_ptr_array_index (parents, i)); - g_ptr_array_free (parents, TRUE); - - from = g_mime_message_get_sender (message); - addresses = internet_address_list_parse_string (from); - - add_terms_address_addrs (doc, addresses, "from_email"); - - add_terms_address_addrs (doc, - g_mime_message_get_all_recipients (message), - "to_email"); - - g_mime_message_get_date (message, &time, NULL); - - gmtime_r (&time, &gm_time_tm); - - if (strftime (date_str, sizeof (date_str), - "%Y%m%d%H%M%S", &gm_time_tm) == 0) { - fprintf (stderr, "Internal error formatting time\n"); - exit (1); - } - - add_term (doc, "date", date_str); - - add_term (doc, "label", "inbox"); - add_term (doc, "label", "unread"); - add_term (doc, "type", "mail"); - add_term (doc, "source_id", "1"); - - if (message_id) { - add_term (doc, "msgid", message_id); - doc.add_value (NOTMUCH_VALUE_MESSAGE_ID, message_id); - } - - if (thread_ids->len) { - unsigned int i; - GString *thread_id; - char *id; - - for (i = 0; i < thread_ids->len; i++) { - id = (char *) thread_ids->pdata[i]; - - add_term (doc, "thread", id); - - if (i == 0) - thread_id = g_string_new (id); - else - g_string_append_printf (thread_id, ",%s", id); - - free (id); - } - g_ptr_array_free (thread_ids, TRUE); - - doc.add_value (NOTMUCH_VALUE_THREAD, thread_id->str); - - g_string_free (thread_id, TRUE); - } else if (message_id) { - /* If not part of any existing thread, generate a new thread_id. */ - thread_id_t thread_id; - - thread_id_generate (&thread_id); - - add_term (doc, "thread", thread_id.str); - doc.add_value (NOTMUCH_VALUE_THREAD, thread_id.str); - } - - doc.add_value (NOTMUCH_VALUE_DATE, Xapian::sortable_serialise (time)); - - db.add_document (doc); - - g_object_unref (message); - g_object_unref (parser); - g_object_unref (stream); -} - -static void -usage (const char *argv0) -{ - fprintf (stderr, "Usage: %s <path-to-xapian-database>\n", argv0); - fprintf (stderr, "\n"); - fprintf (stderr, "Messages to be indexed are read from stdnin as absolute filenames\n"); - fprintf (stderr, "one file per line."); -} - -int -main (int argc, char **argv) -{ - const char *database_path; - char *filename; - GIOChannel *channel; - GIOStatus gio_status; - GError *error = NULL; - int count; - struct timeval tv_start, tv_last, tv_now; - double elapsed; - - if (argc < 2) { - usage (argv[0]); - exit (1); - } - - database_path = argv[1]; - - g_mime_init (0); - - try { - Xapian::WritableDatabase db; - Xapian::TermGenerator term_gen; - - db = Xapian::WritableDatabase (database_path, - Xapian::DB_CREATE_OR_OPEN); - - term_gen = Xapian::TermGenerator (); - - channel = g_io_channel_unix_new (fileno (stdin)); - - count = 0; - - gettimeofday (&tv_start, NULL); - tv_last = tv_start; - - while (1) { - gio_status = g_io_channel_read_line (channel, &filename, - NULL, NULL, &error); - if (gio_status == G_IO_STATUS_EOF) - break; - if (gio_status != G_IO_STATUS_NORMAL) { - fprintf (stderr, "An error occurred reading from stdin: %s\n", - error->message); - exit (1); - } - - g_strchomp (filename); - index_file (db, term_gen, filename); - - g_free (filename); - - count++; - if (count % 1000 == 0) { - gettimeofday (&tv_now, NULL); - printf ("Indexed %d messages (%g messages/second)\n", - count, 1000 / ((tv_now.tv_sec - tv_last.tv_sec) + - (tv_now.tv_usec - tv_last.tv_usec) / 1e6)); - tv_last = tv_now; - } - } - - g_io_channel_unref (channel); - - gettimeofday (&tv_now, NULL); - elapsed = (tv_now.tv_sec - tv_start.tv_sec + - (tv_now.tv_usec - tv_start.tv_usec) / 1e6); - printf ("Completed indexing of %d messages in %g seconds (%g messages/second)\n", - count, elapsed, count / elapsed); - - } catch (const Xapian::Error &error) { - cerr << "A Xapian exception occurred: " << error.get_msg () << endl; - exit (1); - } - - return 0; -} diff --git a/xapian-dump.cc b/xapian-dump.cc deleted file mode 100644 index 39c1c340..00000000 --- a/xapian-dump.cc +++ /dev/null @@ -1,204 +0,0 @@ -/* xapian-dump: Create a textual dump of a Xapian database. - * - * Copyright © 2009 Carl Worth - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see http://www.gnu.org/licenses/ . - * - * Author: Carl Worth <cworth@cworth.org> - */ - -/* Currently the dumped data includes: - * - * All document IDs - * - * And for each document ID: - * - * Document data - * All document terms - * All document values - */ - -#include <cstdlib> -#include <iostream> -#include <algorithm> - -#include <xapian.h> - -using namespace std; - -vector<int> UNSERIALIZE; - -unsigned int MAX_TERMS = 0; - -static void -print_escaped_string (const char *s) -{ - printf ("\""); - - while (*s) { - if (*s == '"') - printf ("\\"); - printf ("%c", *s); - s++; - } - - printf ("\""); -} - -static void -print_document_terms (Xapian::Document doc) -{ - Xapian::TermIterator it; - unsigned int i; - - printf (" {\n"); - - for (it = doc.termlist_begin (), i = 0; - it != doc.termlist_end (); - it++, i++) - { - printf (" "); - print_escaped_string ((*it).c_str()); - printf (",\n"); - } - - for ( ; i < MAX_TERMS; i++) - printf (" \"\",\n"); - - printf (" },\n"); -} - -static int -vector_int_contains (vector<int> v, int i) -{ - vector<int>::iterator result; - - result = find (v.begin(), v.end(), i); - - return result != v.end(); -} - -static void -print_document_values (Xapian::Document doc) -{ - Xapian::ValueIterator i; - int value_no, value_int; - double value_float; - - for (i = doc.values_begin (); i != doc.values_end (); i++) { - value_no = i.get_valueno(); - - printf (" "); - - if (vector_int_contains (UNSERIALIZE, value_no)) { - value_float = Xapian::sortable_unserialise (*i); - value_int = value_float; - if (value_int == value_float) - printf ("%d", value_int); - else - printf ("\"%f\"", value_float); - } else { - print_escaped_string ((*i).c_str ()); - } - - printf (",\n"); - } - -} - -static void -print_document (Xapian::Database db, Xapian::docid id) -{ - Xapian::Document doc; - - printf ("{\n"); - - doc = db.get_document (id); - - printf (" \"%s\",\n", doc.get_data ().c_str()); - - print_document_terms (doc); - - print_document_values (doc); - - printf ("},\n"); -} - -int -main (int argc, char *argv[]) -{ - const char *database_path; - int i; - - if (argc < 2) { - fprintf (stderr, "Usage: %s <path-to-xapian-database> [value_nos...]\n", - argv[0]); - fprintf (stderr, "Dumps data from the given database.\n"); - fprintf (stderr, "The values corresponding to any value numbers given on the command line\n"); - fprintf (stderr, "will be unserialized to an before being printed.\n"); - exit (1); - } - - database_path = argv[1]; - - UNSERIALIZE = vector<int> (); - - for (i = 2; i < argc; i++) - UNSERIALIZE.push_back (atoi (argv[i])); - - try { - Xapian::Database db; - Xapian::PostingIterator i; - Xapian::docid doc_id; - - db = Xapian::Database (database_path); - - for (i = db.postlist_begin (""); i != db.postlist_end (""); i++) { - Xapian::Document doc; - - doc_id = *i; - - doc = db.get_document (doc_id); - - if (doc.termlist_count () > MAX_TERMS) - MAX_TERMS = doc.termlist_count (); - } - - printf ("#define MAX_TERMS %d\n\n", MAX_TERMS); - - printf ("typedef struct {\n" - " char data[255];\n" - " char terms[MAX_TERMS][255];\n" - " char message_id[255];\n" - " char thread_id[4096];\n" - " time_t time;\n" - "} document_dump_t;\n\n"); - - printf ("document_dump_t dump[] = {\n"); - - for (i = db.postlist_begin (""); i != db.postlist_end (""); i++) { - doc_id = *i; - - print_document (db, doc_id); - } - - printf ("};\n"); - - } catch (const Xapian::Error &error) { - cerr << "A Xapian exception occurred: " << error.get_msg () << endl; - exit (1); - } - - return 0; -} |