diff --git a/.gitignore b/.gitignore index a64c2f8..8693584 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ .*.swp .*.swo +mailstore/ +tags +logfile.txt diff --git a/bin/heliotrope-syncback b/bin/heliotrope-syncback new file mode 100644 index 0000000..3a70b08 --- /dev/null +++ b/bin/heliotrope-syncback @@ -0,0 +1,170 @@ +#!/usr/bin/env ruby +# encoding: UTF-8 + +require 'json' +require 'net/imap' +require 'set' + +def remove_words_for_imap_compliance string + # Net::IMAP only wants to send ASCII string. : see imap.rb:1224 + # So we have to remove every character that is not ascii (we cannot + # convert) + # The thing is, Net::IMAP is only used for search. So we have to remove + # entire words were non-ascii characters appear + words_list = string.split + out = "" + words_list.each do |w| + if w.match /\u00e9/u # supposed to be é : remove the word, don't need to parse it any further + next + elsif w.match /\u0092/u # used in a malformed sentence to mean ' + w.gsub! /\u0092/u, "\u0027" + end + out << " " << w + end + + out.strip +end + + +@logfile = "logfile.txt" + + +## these are things that can be set on a per-message basis. each one +## corresponds to a particular label, but labels are propagated at the +## thread level whereas state is not. +MESSAGE_MUTABLE_STATE = Set.new %w(starred unread deleted) +## flags that are set per-message but are not modifiable by the user +MESSAGE_IMMUTABLE_STATE = Set.new %w(attachment signed encrypted draft sent) +MESSAGE_STATE = MESSAGE_MUTABLE_STATE + MESSAGE_IMMUTABLE_STATE + + + + +if File.exists? @logfile + logfile = File.open( @logfile,"r+") +else + logfile = File.new( @logfile,"r+") +end +puts "-- Reading operations on #{Time.now}" + +puts "-- connecting to GMail" +imap = Net::IMAP.new("imap.gmail.com","993",true) +imap.login("salutlesamis.coucou@gmail.com", "coucoupassword") +puts "-- connected to GMail" + +logfile.each do |line| + hash = JSON.parse line + subject = hash["subject"] + + subject = remove_words_for_imap_compliance subject + labels_to_add = Set.new hash["labels_to_add"] + labels_to_remove = Set.new hash["labels_to_remove"] + states_to_add = Set.new hash["states_to_add"] + states_to_remove = Set.new hash["states_to_remove"] + puts "-- Work on [#{subject}]" + +# labels shouldn't contain message states ? + labels_to_add = labels_to_add - MESSAGE_MUTABLE_STATE + labels_to_remove = labels_to_remove - MESSAGE_MUTABLE_STATE + +# add label to thread +# except if label is one of MESSAGE_MUTABLE_STATE + if !labels_to_add.empty? + labels_to_add.each do |label| + if not imap.list('', label) + puts " create label #{label} on remote" + imap.create(label) + end + imap.select('[Gmail]/All Mail') #All mails are in "All Mail"; do ops from here + # Here's the weak part : messages are only propagated if the search on + # the remote server is successful => problems with malformed "subject" + # strings + imap.uid_search(["SUBJECT", subject]).each do |message_uid| + puts " add label #{label} on message #{message_uid}" + imap.uid_copy message_uid, label + # Note : Gmail does support IMAP keywords (these : + # http://deflexion.com/2006/05/server-side-message-labels) + # but they are not used by Gmail in any way -- especially, they + # are not visible on the webmail UI -- so we don't update this field, although it would be preferrable + end + end + end + + +#add state to thread + if !states_to_add.empty? + states_to_add.each do |state| + if state == "starred" # Starred is to be dealt with like a label + if not imap.list('', "[Gmail]/Starred") + puts " create label [Gmail]/Starred on remote" + imap.create("[Gmail]/Starred") + end + imap.select('[Gmail]/All Mail') + # Problem : we can't search for a specific message, unless we search for + # the whole content of each message. So we star the whole label + imap.uid_search(["SUBJECT", subject]).each do |message_uid| + puts " star message #{message_uid}" + imap.uid_copy message_uid, "[Gmail]/Starred" + end + elsif state == "unread" + # Same problem here : we have to apply the treatment to the whole thread + imap.uid_search(["SUBJECT", subject]).each do |message_uid| + puts " mark message #{message_uid} as unread" + imap.uid_store message_uid, "-FLAGS", [:Seen] + end + elsif state == "deleted" + # I don't want to delete my mails, so I don't use this + end + end + end + + +# remove label from thread +# except if label is one of MESSAGE_MUTABLE_STATE + if !labels_to_remove.empty? + labels_to_remove.each do |label| + if not imap.list('', label) + puts "error in thread : label #{label} doesn't exist" + end + imap.select label + imap.uid_search(["SUBJECT", subject]).each do |message_uid| + puts " delete label #{label} on message #{message_uid}" + imap.uid_store message_uid, "+FLAGS", [:Deleted] + end + imap.expunge + end + end + + +#remove state to thread + if !states_to_remove.empty? + states_to_remove.each do |state| + if state == "starred" # Starred is to be dealt with like a label + imap.select('[Gmail]/Starred') + imap.uid_search(["SUBJECT", subject]).each do |message_uid| + puts " unstar message #{message_uid}" + imap.uid_store message_uid, "+FLAGS", [:Deleted] + end + imap.expunge + elsif state == "unread" + # Same problem here : we have to apply the treatment to the whole thread + imap.uid_search(["SUBJECT", subject]).each do |message_uid| + puts " mark message #{message_uid} as unread" + imap.uid_store message_uid, "+FLAGS", [:Seen] + end + elsif state == "deleted" + # I don't want to delete my mails, so I don't use this + end + end + end + + +end + +imap.logout +imap.disconnect +puts "-- disconnected from GMail" + +logfile.close + + diff --git a/lib/heliotrope-client.rb b/lib/heliotrope-client.rb index 5499735..f6196d6 100644 --- a/lib/heliotrope-client.rb +++ b/lib/heliotrope-client.rb @@ -57,7 +57,9 @@ class HeliotropeClient def info; get_json("info") end def size; get_json("size")["size"] end - def prune_labels!; post_json("labels/prune")["labels"] end + def prune_labels! + post_json("labels/prune")["labels"] + end def set_labels! thread_id, labels post_json "thread/#{thread_id}/labels", :labels => labels.to_json diff --git a/lib/heliotrope/meta-index.rb b/lib/heliotrope/meta-index.rb index 980ad5b..e1ee0af 100644 --- a/lib/heliotrope/meta-index.rb +++ b/lib/heliotrope/meta-index.rb @@ -3,6 +3,7 @@ require 'whistlepig' require 'leveldb' require 'set' +require 'json' class Array def ordered_uniq @@ -41,8 +42,19 @@ class MetaIndex @index = index @hooks = hooks @query = nil # we always have (at most) one active query - @debug = false + @debug = true reset_timers! + + @logfile = "logfile.txt" + + if File.exists? @logfile + logfile = File.open( @logfile,"a") + else + logfile = File.new( @logfile,"a") + end + + logfile.close + end def close @@ -172,7 +184,6 @@ class MetaIndex old_tlabels = load_set key new_tlabels = (old_tlabels & MESSAGE_STATE) + labels write_set key, new_tlabels - threadinfo = load_hash "thread/#{threadid}" write_thread_message_labels! threadinfo[:structure], new_tlabels @@ -308,6 +319,29 @@ private changed = new_mstate != old_mstate write_set key, new_mstate if changed + + # Write changes to logfile to sync back to imap server + key = "doc/#{docid}" + hash = load_hash key + subject = hash.fetch :subject + + logfile = File.open( @logfile,"a") + states_to_add = Array.new + states_to_remove = Array.new + (old_mstate - new_mstate).each { |l| states_to_remove << l } + (new_mstate - old_mstate ).each { |l| states_to_add << l} + data = { + :message_id => docid, + :subject => subject, + :states_to_add => states_to_add, + :states_to_remove => states_to_remove, + :date => Time.now + } + string = JSON.generate data + logfile.puts string + + logfile.close + [changed, new_mstate] end @@ -410,6 +444,29 @@ private puts "; adding ~#{l} to #{docid}" if @debug @index.add_label docid, l end + + # Write changes to logfile to sync back to imap server + + key = "doc/#{docid}" + hash = load_hash key + subject = hash.fetch :subject + logfile = File.open( @logfile,"a") + labels_to_add = Array.new + labels_to_remove = Array.new + (oldlabels - labels).each { |l| labels_to_remove << l } + (labels - oldlabels ).each { |l| labels_to_add << l} + data = { + :message_id => docid, + :subject => subject, + :labels_to_add => labels_to_add, + :labels_to_remove => labels_to_remove, + :date => Time.now + } + string = JSON.generate data + logfile.puts string + logfile.close + + end end