This is an automated email from the ASF dual-hosted git repository. sebb pushed a commit to branch mod-gui in repository https://gitbox.apache.org/repos/asf/whimsy.git
commit 264df506941619da9500829a7d9dcedf44bd62c8 Author: Sebb <[email protected]> AuthorDate: Wed Jan 9 15:22:49 2019 +0000 www/moderation/ Initial publish --- www/moderation/desk/.gitignore | 2 + www/moderation/desk/Gemfile | 27 ++ www/moderation/desk/README | 70 ++++ www/moderation/desk/Rakefile | 54 +++ www/moderation/desk/TODO.txt | 20 ++ www/moderation/desk/config.rb | 8 + www/moderation/desk/config.ru | 11 + www/moderation/desk/defines.rb | 7 + www/moderation/desk/deliver.rb | 89 +++++ www/moderation/desk/helpers.rb | 37 +++ www/moderation/desk/models/attachment.rb | 87 +++++ www/moderation/desk/models/events.rb | 72 ++++ www/moderation/desk/models/mailbox.rb | 282 ++++++++++++++++ www/moderation/desk/models/message.rb | 365 +++++++++++++++++++++ www/moderation/desk/models/safetemp.rb | 28 ++ .../desk/public/assets/bootstrap-min.css | 6 + www/moderation/desk/public/assets/bootstrap-min.js | 7 + www/moderation/desk/public/assets/jquery-min.js | 4 + www/moderation/desk/public/assets/vue.min.js | 6 + www/moderation/desk/public/secmail.css | 114 +++++++ www/moderation/desk/public/transparent.png | Bin 0 -> 1006 bytes www/moderation/desk/server.rb | 198 +++++++++++ www/moderation/desk/templates/reject-off-topic.erb | 7 + www/moderation/desk/views/actions/email.json.rb | 72 ++++ www/moderation/desk/views/app.js.rb | 10 + www/moderation/desk/views/body.html.rb | 124 +++++++ www/moderation/desk/views/danger.html.rb | 21 ++ www/moderation/desk/views/headers.html.rb | 7 + www/moderation/desk/views/http.js.rb | 135 ++++++++ www/moderation/desk/views/main.html.rb | 14 + www/moderation/desk/views/messages.html.rb | 27 ++ www/moderation/desk/views/messages.js.rb | 321 ++++++++++++++++++ www/moderation/desk/views/status.js.rb | 53 +++ www/moderation/desk/views/vue-config.js.rb | 10 + 34 files changed, 2295 insertions(+) diff --git a/www/moderation/desk/.gitignore b/www/moderation/desk/.gitignore new file mode 100644 index 0000000..169b1cf --- /dev/null +++ b/www/moderation/desk/.gitignore @@ -0,0 +1,2 @@ +Gemfile.lock +officers-secretary diff --git a/www/moderation/desk/Gemfile b/www/moderation/desk/Gemfile new file mode 100644 index 0000000..b7893aa --- /dev/null +++ b/www/moderation/desk/Gemfile @@ -0,0 +1,27 @@ +source 'https://rubygems.org' + +root = '../../../..' +version_file = File.expand_path("#{root}/asf.version", __FILE__) +if File.exist? version_file + # for deployment and local testing + asf_version = File.read(version_file).chomp + gem 'whimsy-asf', asf_version, path: File.expand_path(root, __FILE__) +else + # for docker purposes (atleast for now) + gem 'whimsy-asf' +end + +gem 'mail' +gem 'rake' +gem 'zip' +gem 'sinatra', '~> 2.0' +gem 'sanitize' +gem 'wunderbar' +gem 'ruby2js', '>= 2.1.18' +gem 'execjs' +gem 'listen', ('~> 3.0.7' if RUBY_VERSION =~ /^2\.[01]/) +gem 'escape' + +group :demo do + gem 'puma' +end diff --git a/www/moderation/desk/README b/www/moderation/desk/README new file mode 100644 index 0000000..2950859 --- /dev/null +++ b/www/moderation/desk/README @@ -0,0 +1,70 @@ +This is based loosely on the secretary/workbench (q.v.) + +To set up for testing: + +create archive directory /srv/mail/moderation +Note that the files in it must be readable by the web server, and the .yml files must be writable. + +Populate the database by running some sample mails through +www/moderation/desk/deliver.rb +This should create yyyymmdd.yml and yyyymmdd.mail/* +(ensure *.yml are writable by webserver afterwards) + +Add the following to the web server setup: + + Alias /moderation/desk/ /srv/whimsy/www/moderation/desk/public + + <Location /moderation/desk> + PassengerBaseURI /moderation/desk + PassengerAppRoot /srv/whimsy/www/moderation/desk + Options -MultiViews + CheckSpelling Off + # SetEnv HTTPS on + </Location> + +Navigate to http://localhost/moderation/desk + +Design Notes +============ +The Secretary Workbench has to deal with a few emails per day and a few users. +It uses a file per month for the indexes, and a directoy per month for the raw mails. + +There are about as many moderation emails per day as Secretary emails per month. +Also ezmlm will not process responses after about 10 days. +The initial implementation therefore uses 1 file per day. +The file access is confined to the mailbox.rb file, so the storage could be changed +if necessary - e.g. a database. The mailbox.rb file is also responsible for access control. + +The existing moderation process involves replying to a mail. Whilst this only takes a couple +of clicks (except perhaps for rejects), the volume of mails can be very large, so it is +important to minimise navigation and user actions, especially for the commonest cases. + +The initial approach is to show a list of emails (oldest at the top) with buttons for +Accept/Allow/Reject/Mark Spam. Apart from Reject, no further input is needed. +As soon as an email has been dealt with, it should disappear from all the screens. + +This reduces the workload for moderators, because each message only needs to be dealt with once. + +Spam +==== +By far the largest number of mails are spam; a significant proportion of these are very similar. +At present each message would have to be dealt with separately (though only by one person). It would be +good if similar mails could be automatically marked. The challenge is to guard against false positives. +One idea (yet to be implemented) is to count the number of mails marked as spam that have the same: +From: Return-Path: Subject: and possibly other headers. +When a few mails have been seen, other mails with the same headers can potentially be flagged as +likely spam. However, care must be taken not to mark too many mails - e.g. some subjects may be +used by valid mails. It might be best to gather some more data before proceeding with automation. + +[Even without this, the workload should be considerably reduced. Also the greatest proportion of +spam is unfortunately not readily detectable] + + +Authentication +============== +TBA + +Filtering +========= +It's best if moderators are familiar with the subject matter of the lists that they moderate, +so they should only be shown mails for their area. The simplest might be to use PMC membership. \ No newline at end of file diff --git a/www/moderation/desk/Rakefile b/www/moderation/desk/Rakefile new file mode 100644 index 0000000..d5d0564 --- /dev/null +++ b/www/moderation/desk/Rakefile @@ -0,0 +1,54 @@ +require_relative 'config' + +verbose false + +task :default do + puts 'Usage:' + sh 'rake', '-T' +end + +file 'Gemfile.lock' => 'Gemfile' do + sh 'bundle update' + touch 'Gemfile.lock' +end + +desc 'install dependencies' +task :bundle => 'Gemfile.lock' + +desc 'Parse emails' +task :parse => :bundle do + ruby 'parsemail.rb' +end + +desc 'create /srv/mail with the appropriate permissions' +file '/srv/mail' do + begin + mkdir_p '/srv/mail' + + require 'etc' + if Etc.getpwent.uid == 0 + user = Etc.getpwnam(Etc.getlogin) + chown user.uid, user.gid, '/srv/mail' + end + rescue Errno::EACCES + sh 'sudo rake /srv/mail' + end +end + +desc 'WebServer that provides an interface to explore emails' +task :server => :bundle do + ENV['RACK_ENV']='development' + + require 'bundler/setup' + require 'wunderbar' + module Wunderbar::Listen + EXCLUDE = [ARCHIVE] + end + + require 'wunderbar/listen' +end + +desc 'remove all parsed yaml files' +task :clean do + rm_rf Dir["#{ARCHIVE}/*.yml"] +end diff --git a/www/moderation/desk/TODO.txt b/www/moderation/desk/TODO.txt new file mode 100644 index 0000000..5aea0bd --- /dev/null +++ b/www/moderation/desk/TODO.txt @@ -0,0 +1,20 @@ +Authorisation +- started; hooks added + +Filtering of lists +- users may not want to see all the lists to which they are entitled +- hooks added + +Implement Actions: +- accept and allow +- single accept +- reject with reply e.g. as + * off topic + * confidential material + [Started] +- reply to subscriber confirm asking to update LDAP + +Implement message templates for rejection etc + +Better GUI layout += e.g. fixed header with scrolling message list beneath \ No newline at end of file diff --git a/www/moderation/desk/config.rb b/www/moderation/desk/config.rb new file mode 100644 index 0000000..2632c9c --- /dev/null +++ b/www/moderation/desk/config.rb @@ -0,0 +1,8 @@ +# +# Where to find the archive +# + +ARCHIVE = '/srv/mail/moderation' + +MBOX_RE='\d{8}' # yyyymmdd +HASH_RE='[a-f0-9]+' # perhaps fix length? diff --git a/www/moderation/desk/config.ru b/www/moderation/desk/config.ru new file mode 100644 index 0000000..c3157de --- /dev/null +++ b/www/moderation/desk/config.ru @@ -0,0 +1,11 @@ +require File.expand_path('../server.rb', __FILE__) + +require 'whimsy/asf/rack' + +use ASF::HTTPS_workarounds +use ASF::Auth::MembersAndOfficers +use ASF::AutoGC + +use ASF::DocumentRoot + +run Sinatra::Application diff --git a/www/moderation/desk/defines.rb b/www/moderation/desk/defines.rb new file mode 100644 index 0000000..521c22b --- /dev/null +++ b/www/moderation/desk/defines.rb @@ -0,0 +1,7 @@ +# values shared between Javascript and server code + +ACCEPT=':Accept' +ACCEPTALLOW=':AcceptAllow' +# does not make much sense to allow on its own +REJECT=':Reject' +MARKSPAM=':Spam' \ No newline at end of file diff --git a/www/moderation/desk/deliver.rb b/www/moderation/desk/deliver.rb new file mode 100644 index 0000000..e574922 --- /dev/null +++ b/www/moderation/desk/deliver.rb @@ -0,0 +1,89 @@ +# +# Process email as it is received +# + +#Dir.chdir File.dirname(File.expand_path(__FILE__)) + +require_relative 'models/mailbox' +require 'mail' +require_relative 'config.rb' + +# read and parse email +STDIN.binmode +original = STDIN.read +hash = Message.hash(original) + +fail = nil +mailbox = nil + +mbox=Mailbox.mboxname(Time.now) # for mails that don't have the stamp + +begin + mail = Mail.read_from_string(original) + parts = mail.parts + subj = mail.subject || '' + # other methods give to, cc etc as arrays + # Date is an object, not sure how to ge + + if parts.length == 2 and parts[1].content_type == 'message/rfc822' and subj.start_with? 'MODERATE for ' + hdrs = Mailbox.headers(mail) + # e.g. Subject: MODERATE for [email protected] + list, dom = subj.sub(/^MODERATE for /,'').split'@' + # e.g. reply-To: [email protected] + rp = hdrs['Reply-To'] + timestamp = rp[/-(\d+\.\d+)\./, 1] + headers = { + allow: hdrs['Cc'], # parser uses these standard names, regardless of input capitalisation + accept: rp, + timestamp: timestamp, + reject: hdrs['From'], + list: list, + domain: dom, + date: hdrs['Date'], + } + mbox=Mailbox.mboxname(timestamp.to_f) + # construct wrapper message + mailbox = Mailbox.new(mbox) + # change the hash to change the name of the saved mail + message = Message.new(mailbox, "#{hash}.orig", nil, original) + + # write message to disk + File.umask(0002) + # skip message.write_headers as don't want the headers in a separate file + message.write_email + # extract the message for main mailbox + email = parts[1].body.raw_source + elsif subj.start_with? 'CONFIRM subscribe to ' + headers = Hash.new + email = original + else + $stderr.puts "Unexpected moderation message in #{hash} with subject: #{subj}" + headers = Hash.new + email = original + end + WANTED=%w{Date From To Reply-To Message-ID Subject Return-Path Sender References In-Reply-To} + headers.merge! Message.parse(email).select{ |k,v| Symbol === k or WANTED.include? k } +rescue => e + fail = e + headers = { + exception: e.to_s, + backtrace: e.backtrace[0], + message: 'See procmail.log for full details' + } +end + +# construct message +mailbox = Mailbox.new(mbox) unless mailbox # reuse if exists +message = Message.new(mailbox, hash, headers, email) + +# write message to disk +File.umask(0002) +message.write_headers +message.write_email + +# Now fail if there was an error +if fail + require 'time' + $stderr.puts "WARNING: #{Time.now.utc.iso8601}: error processing email with hash: #{hash}" + raise fail +end diff --git a/www/moderation/desk/helpers.rb b/www/moderation/desk/helpers.rb new file mode 100644 index 0000000..ce77ad1 --- /dev/null +++ b/www/moderation/desk/helpers.rb @@ -0,0 +1,37 @@ +helpers do + # replace inline images (cid:) with references to attachments + def fixup_images(node) + if Wunderbar::Node === node + if node.name == 'img' + src = node.attrs['src'] + if src + if src.to_s.start_with? 'cid:' + src.value = src.to_s.sub('cid:', '') + else # src.to_s.start_with? 'http' # Don't allow access to remote images + src.value='../../transparent.png' + end + end + else + fixup_images(node.search('img')) + end + elsif Array === node + node.each {|child| fixup_images(child)} + end + end +end + + +class Wunderbar::JsonBuilder + # + # extract/verify project (set @pmc and @podling) + # + + # update the status of a message + def _status(status_text) + message = Mailbox.find(@message) + message.headers[:secmail] ||= {} + message.headers[:secmail][:status] = status_text + message.write_headers + _headers message.headers + end +end diff --git a/www/moderation/desk/models/attachment.rb b/www/moderation/desk/models/attachment.rb new file mode 100644 index 0000000..f5643ad --- /dev/null +++ b/www/moderation/desk/models/attachment.rb @@ -0,0 +1,87 @@ +class Attachment + IMAGE_TYPES = %w(.gif, .jpg, .jpeg, .png) + attr_reader :headers + + def initialize(message, headers, part) + @message = message + @headers = headers + @part = part + end + + def name + headers[:name] || @part.filename + end + + def content_type + type = headers[:mime] || @part.content_type + + if type == 'application/octet-stream' or type == 'text/plain' + type = 'text/plain' if name.end_with? '.sig' + type = 'text/plain' if name.end_with? '.asc' + type = 'application/pdf' if name.end_with? '.pdf' + type = 'image/gif' if name.end_with? '.gif' + type = 'image/jpeg' if name.end_with? '.jpg' + type = 'image/jpeg' if name.end_with? '.jpeg' + type = 'image/png' if name.end_with? '.png' + end + + type = "image/#{$1}" if type =~ /^application\/(jpeg|gif|png)$/ + + type + end + + def body + headers[:content] || @part.body + end + + def safe_name + name = self.name.dup + name.gsub! /^\W/, '' + name.gsub! /[^\w.]/, '_' + name.untaint + end + + def as_file + file = SafeTempFile.new([safe_name, '.pdf']) + file.write(body) + file.rewind + file + end + + def as_pdf + ext = File.extname(name).downcase + ext = '.pdf' if content_type.end_with? '/pdf' + ext.untaint if ext =~ /^\.\w+$/ + + file = SafeTempFile.new([safe_name, ext]) + file.write(body) + file.rewind + + return file if ext == '.pdf' + + if IMAGE_TYPES.include? ext or content_type.start_with? 'image/' + pdf = SafeTempFile.new([safe_name, '.pdf']) + img2pdf = File.expand_path('../img2pdf', __dir__.untaint).untaint + system img2pdf, '--output', pdf.path, file.path + file.unlink + raise "Failed to convert #{self.name} to PDF" unless File.size? pdf.path + return pdf + end + + return file + end + + # write a file out to svn + def write_svn(repos, file, path=nil) + filename = File.join(repos, file) + filename = File.join(filename, path || safe_name) if Dir.exist? filename + + raise Errno::EEXIST.new(file) if File.exist? filename + File.write filename, body, encoding: Encoding::BINARY + + system 'svn', 'add', filename + system 'svn', 'propset', 'svn:mime-type', content_type.untaint, filename + + filename + end +end diff --git a/www/moderation/desk/models/events.rb b/www/moderation/desk/models/events.rb new file mode 100644 index 0000000..52a8656 --- /dev/null +++ b/www/moderation/desk/models/events.rb @@ -0,0 +1,72 @@ +require 'listen' +require 'thread' + +class Events + @@list = [] + + def initialize + @@list.push self + + @events = Queue.new + + @listener = Listen.to ARCHIVE do |modified, added, removed| + (modified + added).each do |file| + next unless file.end_with? '.yml' + mbox = Mailbox.new(File.basename(file)) +# $stderr.puts "Event #{mbox}" + @events.push({messages: mbox.client_headers}) + end + end + + @listener.start + + @closed = false + + # As some TCP/IP implementations will close idle sockets after as little + # as 30 seconds, sent out a heartbeat every 25 seconds. Due to limitations + # of some versions of Ruby (2.0, 2.1), this is lowered to every 5 seconds + # in development mode to allow for quicker restarting after a trap/signal. + Thread.new do + loop do + sleep(ENV['RACK_ENV'] == 'development' ? 5 : 25) + break if @closed + @events.push(:heartbeat) + end + + @events.push(:exit) + @listener.stop + end + end + + def pop + @events.pop + end + + def close + @@list.delete self + @events.clear + @closed = true + + begin + @events.push :exit + rescue ThreadError + # some versions of Ruby don't allow queue operations in traps + end + end + + def self.shutdown + @@list.dup.each {|event| event.close} + end +end + +# puma uses SIGUSR2 +restart_usr2 ||= trap 'SIGUSR2' do + restart_usr2.call if Proc === restart_usr2 + Events.shutdown +end + +# thin uses SIGHUP +restart_hup ||= trap 'SIGHUP' do + restart_hup.call if Proc === restart_hup + Events.shutdown +end diff --git a/www/moderation/desk/models/mailbox.rb b/www/moderation/desk/models/mailbox.rb new file mode 100644 index 0000000..e7fcf1a --- /dev/null +++ b/www/moderation/desk/models/mailbox.rb @@ -0,0 +1,282 @@ +# +# Encapsulate access to mailboxes +# + +# It may be necessary to use a database rather than files, so try to avoid exposing +# any of the internal storage details + +require 'zlib' +require 'zip' +require 'stringio' +require 'yaml' + +require_relative '../config.rb' + +require_relative 'message.rb' + +class Mailbox + + # + # Initialize a mailbox + # + def initialize(name) + name = File.basename(name, '.yml') + + if name =~ /^#{MBOX_RE}$/ + @name = name.untaint + @mbox = Dir["#{ARCHIVE}/#{@name}", "#{ARCHIVE}/#{@name}.gz"].first.untaint + else + @name = name.split('.').first + @mbox = "#{ARCHIVE}/#{name}" + end + end + + # centralise the name generation + def self.mboxname(timestamp=nil) + # If nil, return the current (last) box (could perhaps be the first) + Time.at(timestamp||Time.now).gmtime.strftime('%Y%m%d') + end + + # + # convenience interface to update status + # + def self.status!(name, hash, newstatus) + success = false + Mailbox.new(name).update do |headers| + target = headers[hash] + return unless Mailbox.message_visible?(target) # TODO should this throw? + if target + # TODO don't allow same message to be counted twice (perhaps store ids in spam lists) + # TODO capture spammy headers if marking as spam + # TODO skip update if no record found or no change made +# if newstatus == :spam +# end + target[:status] = newstatus + success = true + end + end + success # let caller know + end + + # + # Allow update of entry using hash + # + def self.patch!(name, hash, updates) + success = false + Mailbox.new(name).update do |headers| + target = headers[hash] + return unless Mailbox.message_visible?(target) # TODO should this throw? + if target + # special processing for entries which use symbols as keys + # (allow for :status not present) + [target.keys,:status].flatten.each do |key| + if Symbol === key and updates.has_key? key.to_s + target[key] = updates.delete(key.to_s) # apply the change and drop from input + end + end + + target.merge! updates # anything else can just be merged + success = true + end + end + success # let caller know + end + + def write_email(hash, email) + Dir.mkdir dir, 0755 unless Dir.exist? dir + File.write File.join(dir, hash), email, encoding: Encoding::BINARY + end + + def write_headers(hash, headers) + update do |yaml| + # TODO check if hash exists and throw if so? + # (would need to override this for testing) + yaml[hash] = headers + end + end + + # + # encapsulate updates to a mailbox + # TODO would like to make this private/protected to avoid external updates + def update + File.open(yaml_file, File::RDWR|File::CREAT, 0644) do |file| + file.flock(File::LOCK_EX) + mbox = YAML.load(file.read) || {} rescue {} + yield mbox # TODO allow block to cancel update + file.rewind + file.write YAML.dump(mbox) + file.truncate(file.pos) + end + end + + # + # Find a message by id e.g. yyyymmdd/abcdefgh (for use by GUI) + # Returns nil if not found or no access + def self.find(id) + return unless id + # Allow leading and trailing slash + id.match(%r{^/?(#{MBOX_RE})/(#{HASH_RE})/?$}) do |m| + mbox, hash = m.captures + Mailbox.new(mbox.untaint).find(hash.untaint) + end + end + + + # + # Find headers by id e.g. yyyymmdd/abcdefgh (for use by GUI) + # Returns nil if not found or no access + def self.hdrs(id) + return unless id + # Allow leading and trailing slash + id.match(%r{^/?(#{MBOX_RE})/(#{HASH_RE})/?$}) do |m| + mbox, hash = m.captures + Mailbox.new(mbox.untaint).headers(hash.untaint) + end + end + + # + # Find message headers + # + def headers(hash) + headers = YAML.load_file(yaml_file) rescue return + target = headers[hash] + return unless Mailbox.message_visible? target # don't allow access to private data + target + end + + # + # Find a message + # + def find(hash) + headers = YAML.load_file(yaml_file) rescue {} + return unless Mailbox.message_visible? headers[hash] # don't allow access to private data + + file = File.join(dir, hash) + # TODO why check the dir? + if Dir.exist? dir and File.exist? file + email = File.read(file, encoding: Encoding::BINARY) + end + + Message.new(self, hash, headers[hash], email) if email + end + + + # + # Find the source message; return the raw data + # + def orig(hash) + headers = YAML.load_file(yaml_file) rescue {} + return unless Mailbox.message_visible? headers[hash] # don't allow access to private data + + file = File.join(dir, hash + '.orig') + File.read(file, encoding: Encoding::BINARY) if File.exist? file + end + + # Is the list wanted by the caller? + def list_wanted?(message) + true + end + + # Is the message visible to the caller? + # e.g. private and security lists are generally not visible to all + def self.message_visible?(message) + message and not %w(private security).include? message[:list] # TODO this is just a test + end + + def message_active?(status) + status == nil or status == '' + end + + # + # return headers (client view) + # + def client_headers + # fetch a list of headers for all messages in the mailbox + messages = YAML.load_file(yaml_file) rescue {} + headers = messages.to_a.select do |id, message| + message_active?(message[:status]) && Mailbox.message_visible?(message) && list_wanted?(message) + end + + # extract relevant fields from the headers + headers.map! do |id, message| + { + id: id, + timestamp: message[:timestamp], + list: message[:list], + domain: message[:domain], + allow: message[:allow], + accept: message[:accept], + reject: message[:reject], + return_path: message['Return-Path'], + from: message['From'], + subject: message['Subject'], + status: message[:status], + date: message['Date'] || '', + } + end + + # Look for next box (currently previous day) + # TODO no need to do this for events where a box has been updated + nextmbox = nil + # find the most recent 10 daily files + available = Dir["#{ARCHIVE}/*.yml"].map{|f| File.basename(f, '.yml')}.select{|f| f =~ %r{^#{MBOX_RE}$}}.sort.reverse[0..9] + index = available.find_index {|e| e < @name} # next oldest date from current + if index + nextmbox = available[index].untaint + end + + { + nextmbox: nextmbox, + source: @name, # Same for all headers + headers: headers, + } + end + + # + # common header logic for messages and attachments + # + def self.headers(part) + # extract all fields from the mail (recovering from bad encoding issues) + fields = part.header_fields.map do |field| + begin + next [field.name, field.to_s] if field.to_s.valid_encoding? + rescue + end + + if field.value and field.value.valid_encoding? + [field.name, field.value] + else + [field.name, field.value.inspect] + end + end + + # group fields by name + fields = fields.group_by(&:first).map do |name, values| + if values.length == 1 + [name, values.first.last] + else + [name, values.map(&:last)] + end + end + + # return fields as a Hash + Hash[fields] + end + + private # these methods expose details of the storage + + # + # name of associated yaml file + # + def yaml_file + "#{ARCHIVE}/#{@name}.yml" + end + + # + # name of associated directory + # + def dir + "#{ARCHIVE}/#{@name}.mail" + end + +end diff --git a/www/moderation/desk/models/message.rb b/www/moderation/desk/models/message.rb new file mode 100644 index 0000000..78814ea --- /dev/null +++ b/www/moderation/desk/models/message.rb @@ -0,0 +1,365 @@ +# +# Encapsulate access to messages +# + +require 'digest' +require 'mail' +require 'time' + +require_relative 'attachment.rb' + +class Message + attr_reader :headers + + SIG_MIMES = %w(application/pkcs7-signature application/pgp-signature) + + # + # create a new message + # + def initialize(mailbox, hash, headers, email) + @hash = hash + @mailbox = mailbox + @headers = headers + @email = email + end + + # + # find an attachment + # + def find(name) + name = name[1..-2] if name =~ /^<.*>$/ # drop enclosing <> if present + name = name[2..-1] if name.start_with? './' + name = name.dup.force_encoding('utf-8') + + headers = (@headers[:attachments] || []).find do |attach| + attach[:name] == name or URI.decode(attach[:name]) == name or + attach['Content-ID'].to_s == '<' + name + '>' + end + + if headers + part = mail.attachments.find do |attach| + attach.filename == name or URI.decode(attach.filename) == name or + attach['Content-ID'].to_s == '<' + name + '>' + end + Attachment.new(self, headers, part) + end + end + + # + # accessors + # + + def mail + @mail ||= Mail.new(@email) + end + + def raw + @email + end + + def id + @headers['Message-ID'] + end + + def date + mail[:date] + end + + def from + mail[:from] + end + + def return_path + mail.return_path + end + + def to + mail[:to] + end + + def cc + @headers[:cc] + end + + def cc=(value) + value=value.split("\n") if String === value + @headers[:cc]=value + end + + def bcc + @headers[:bcc] + end + + def bcc=(value) + value=value.split("\n") if String === value + @headers[:bcc]=value + end + + def subject + mail.subject + end + + def html_part + mail.html_part + end + + def text_part + mail.text_part + end + + def self.attachments(headers) + headers[:attachments] || [] + end + + def attachments + Message.attachments(@headers) + end + + # + # attachment operations: update, replace, delete + # + + def update_attachment name, values + attachment = find(name) + if attachment + attachment.headers.merge! values + write_headers + end + end + + def replace_attachment name, values + attachment = find(name) + if attachment + index = @headers[:attachments].find_index(attachment.headers) + @headers[:attachments][index, 1] = Array(values) + write_headers + end + end + + def delete_attachment name + attachment = find(name) + if attachment + @headers[:attachments].delete attachment.headers + @headers[:status] = :deleted if @headers[:attachments].empty? + write_headers + end + end + + # + # write updated headers to disk + # + def write_headers + @mailbox.write_headers(@hash, @headers) + end + + # + # write email to disk + # + def write_email + @mailbox.write_email(@hash, @email) + end + + # + # write one or more attachments to directory containing an svn checkout + # + def write_svn(repos, filename, *attachments) + # drop all nil and empty values + attachments = attachments.flatten.reject {|name| name.to_s.empty?} + + # if last argument is a Hash, treat it as name/value pairs + attachments += attachments.pop.to_a if Hash === attachments.last + + if attachments.flatten.length == 1 + ext = File.extname(attachments.first).downcase.untaint + find(attachments.first).write_svn(repos, filename + ext) + else + # validate filename + unless filename =~ /\A[a-zA-Z][-.\w]+\z/ + raise IOError.new("invalid filename: #{filename}") + end + + # create directory, if necessary + dest = File.join(repos, filename).untaint + unless File.exist? dest + Dir.mkdir dest + Kernel.system 'svn', 'add', dest + end + + # write out selected attachment + attachments.each do |attachment, basename| + find(attachment).write_svn(repos, filename, basename) + end + + dest + end + end + + # + # Construct a reply message, and in the process merge the email + # address from the original message (from, to, cc) with any additional + # address provided on the call (to, cc, bcc). Remove any duplicates + # that may occur not only due to the merge, but also comparing across + # field types (for example, don't cc an address listed on the to field). + # + # Finally, canonicalize (format) the email addresses and ensure that + # the results aren't marked ask tainted, as the Ruby SMTP library will + # refuse to send to tainted addresses, and in the secretary mail application + # the addresses are expected to come from the mail archive and the + # secretary, both of which can be trusted. + # + def reply(fields) + mail = Mail.new + + # fill in the from address + mail.from = fields[:from] + + # fill in the reply to headers + mail.in_reply_to = self.id + mail.references = self.id + + # fill in the subject from the original email + if self.subject =~ /^re:\s/i + mail.subject = self.subject + elsif self.subject + mail.subject = 'Re: ' + self.subject + elsif fields[:subject] + mail.subject = fields[:subject] + end + + # fill in the subject from the original email + mail.body = fields[:body] + + # gather up the to, cc, and bcc addresses + to = [] + cc = [] + bcc = [] + + # process 'to' addresses on method call + if fields[:to] + Array(fields[:to]).compact.each do |addr| + addr = Message.liberal_email_parser(addr) if addr.is_a? String + next if to.any? {|a| a.address = addr.address} + to << addr + end + end + + # process 'from' addresses from original email + self.from.addrs.each do |addr| + next if to.any? {|a| a.address == addr.address} + if fields[:to] + next if cc.any? {|a| a.address == addr.address} + cc << addr + else + to << addr + end + end + + # process 'to' addresses from original email + if self.to + self.to.addrs.each do |addr| + next if to.any? {|a| a.address == addr.address} + next if cc.any? {|a| a.address == addr.address} + cc << addr + end + end + + # process 'cc' addresses from original email + if self.cc + self.cc.each do |addr| + addr = Message.liberal_email_parser(addr) if addr.is_a? String + next if to.any? {|a| a.address == addr.address} + next if cc.any? {|a| a.address == addr.address} + cc << addr + end + end + + # process 'cc' addresses on method call + if fields[:cc] + Array(fields[:cc]).compact.each do |addr| + addr = Message.liberal_email_parser(addr) if addr.is_a? String + next if to.any? {|a| a.address == addr.address} + next if cc.any? {|a| a.address == addr.address} + cc << addr + end + end + + # process 'bcc' addresses on method call + if fields[:bcc] + Array(fields[:bcc]).compact.each do |addr| + addr = Message.liberal_email_parser(addr) if addr.is_a? String + next if to.any? {|a| a.address == addr.address} + next if cc.any? {|a| a.address == addr.address} + next if bcc.any? {|a| a.address == addr.address} + bcc << addr + end + end + + # reformat and untaint email addresses + mail[:to] = to.map {|addr| addr.format.dup.untaint} + mail[:cc] = cc.map {|addr| addr.format.dup.untaint} unless cc.empty? + mail[:bcc] = bcc.map {|addr| addr.format.dup.untaint} unless bcc.empty? + + # return the resulting email + mail + end + + # get the message ID + def self.getmid(hdrs) + mid = hdrs[/^Message-ID:.*/i] + if mid =~ /^Message-ID:\s*$/i # no mid on the first line + # capture the next line and join them together + mid = hdrs[/^Message-ID:.*\r?\n .*/i].sub(/\r?\n/,'') + end + mid + end + + # + # What to use as a hash for mail + # + def self.hash(message) + Digest::SHA1.hexdigest(getmid(message) || message)[0..9] + end + + # + # parse a message, returning headers + # + def self.parse(message) + mail = Mail.read_from_string(message) + + headers = Mailbox.headers(mail) + + # add in attachments + if mail.attachments.length > 0 + + attachments = mail.attachments.map do |attach| + # replace generic octet-stream with a more specific one + mime = attach.mime_type + if mime == 'application/octet-stream' + filename = attach.filename.downcase + mime = 'application/pdf' if filename.end_with? '.pdf' + mime = 'application/png' if filename.end_with? '.png' + mime = 'application/gif' if filename.end_with? '.gif' + mime = 'application/jpeg' if filename.end_with? '.jpg' + mime = 'application/jpeg' if filename.end_with? '.jpeg' + end + + description = { + name: attach.filename, + length: attach.body.to_s.length, + mime: mime + } + + if description[:name].empty? and attach['Content-ID'] + description[:name] = attach['Content-ID'].to_s + end + + description.merge(Mailbox.headers(attach)) + end + + headers[:attachments] = attachments + end + + headers + end + +end diff --git a/www/moderation/desk/models/safetemp.rb b/www/moderation/desk/models/safetemp.rb new file mode 100644 index 0000000..24bc7e9 --- /dev/null +++ b/www/moderation/desk/models/safetemp.rb @@ -0,0 +1,28 @@ +# +# Tempfile in Ruby 2.3.0 has the unfortunate behavior of returning +# an unsafe path and even blowing up when unlink is called in a $SAFE +# environment. This avoids those two problems, while forwarding all all other +# method calls. +# + +require 'tempfile' + +class SafeTempFile + def initialize *args + args << {} unless args.last.instance_of? Hash + args.last[:encoding] = Encoding::BINARY + @tempfile = Tempfile.new *args + end + + def path + @tempfile.path.untaint + end + + def unlink + File.unlink path + end + + def method_missing symbol, *args + @tempfile.send symbol, *args + end +end diff --git a/www/moderation/desk/public/assets/bootstrap-min.css b/www/moderation/desk/public/assets/bootstrap-min.css new file mode 100644 index 0000000..ed3905e --- /dev/null +++ b/www/moderation/desk/public/assets/bootstrap-min.css @@ -0,0 +1,6 @@ +/*! + * Bootstrap v3.3.7 (http://getbootstrap.com) + * Copyright 2011-2016 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + *//*! normalize.css v3.0.3 | MIT License | github.com/necolas/normalize.css */html{font-family:sans-serif;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%}body{margin:0}article,aside,details,figcaption,figure,footer,header,hgroup,main,menu,nav,section,summary{display:block}audio,canvas,progress,video{display:inline-block;vertical-align:baseline}audio:not([controls]){display:none;height:0}[hidden],template{display:none}a{background-color:transparent}a:active,a:hover{outline:0}abbr [...] +/*# sourceMappingURL=bootstrap.min.css.map */ \ No newline at end of file diff --git a/www/moderation/desk/public/assets/bootstrap-min.js b/www/moderation/desk/public/assets/bootstrap-min.js new file mode 100644 index 0000000..9bcd2fc --- /dev/null +++ b/www/moderation/desk/public/assets/bootstrap-min.js @@ -0,0 +1,7 @@ +/*! + * Bootstrap v3.3.7 (http://getbootstrap.com) + * Copyright 2011-2016 Twitter, Inc. + * Licensed under the MIT license + */ +if("undefined"==typeof jQuery)throw new Error("Bootstrap's JavaScript requires jQuery");+function(a){"use strict";var b=a.fn.jquery.split(" ")[0].split(".");if(b[0]<2&&b[1]<9||1==b[0]&&9==b[1]&&b[2]<1||b[0]>3)throw new Error("Bootstrap's JavaScript requires jQuery version 1.9.1 or higher, but lower than version 4")}(jQuery),+function(a){"use strict";function b(){var a=document.createElement("bootstrap"),b={WebkitTransition:"webkitTransitionEnd",MozTransition:"transitionend",OTransition:" [...] +this.activeTarget=b,this.clear();var c=this.selector+'[data-target="'+b+'"],'+this.selector+'[href="'+b+'"]',d=a(c).parents("li").addClass("active");d.parent(".dropdown-menu").length&&(d=d.closest("li.dropdown").addClass("active")),d.trigger("activate.bs.scrollspy")},b.prototype.clear=function(){a(this.selector).parentsUntil(this.options.target,".active").removeClass("active")};var d=a.fn.scrollspy;a.fn.scrollspy=c,a.fn.scrollspy.Constructor=b,a.fn.scrollspy.noConflict=function(){return [...] \ No newline at end of file diff --git a/www/moderation/desk/public/assets/jquery-min.js b/www/moderation/desk/public/assets/jquery-min.js new file mode 100644 index 0000000..644d35e --- /dev/null +++ b/www/moderation/desk/public/assets/jquery-min.js @@ -0,0 +1,4 @@ +/*! jQuery v3.2.1 | (c) JS Foundation and other contributors | jquery.org/license */ +!function(a,b){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw new Error("jQuery requires a window with a document");return b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){"use strict";var c=[],d=a.document,e=Object.getPrototypeOf,f=c.slice,g=c.concat,h=c.push,i=c.indexOf,j={},k=j.toString,l=j.hasOwnProperty,m=l.toString,n=m.call(Object),o={};function p(a,b){b=b||d;var c=b.createElem [...] +a.removeEventListener("load",S),r.ready()}"complete"===d.readyState||"loading"!==d.readyState&&!d.documentElement.doScroll?a.setTimeout(r.ready):(d.addEventListener("DOMContentLoaded",S),a.addEventListener("load",S));var T=function(a,b,c,d,e,f,g){var h=0,i=a.length,j=null==c;if("object"===r.type(c)){e=!0;for(h in c)T(a,b,h,c[h],!0,f,g)}else if(void 0!==d&&(e=!0,r.isFunction(d)||(g=!0),j&&(g?(b.call(a,d),b=null):(j=b,b=function(a,b,c){return j.call(r(a),c)})),b))for(;h<i;h++)b(a[h],c,g?d: [...] +null==d?void 0:d))},attrHooks:{type:{set:function(a,b){if(!o.radioValue&&"radio"===b&&B(a,"input")){var c=a.value;return a.setAttribute("type",b),c&&(a.value=c),b}}}},removeAttr:function(a,b){var c,d=0,e=b&&b.match(L);if(e&&1===a.nodeType)while(c=e[d++])a.removeAttribute(c)}}),lb={set:function(a,b,c){return b===!1?r.removeAttr(a,c):a.setAttribute(c,c),c}},r.each(r.expr.match.bool.source.match(/\w+/g),function(a,b){var c=mb[b]||r.find.attr;mb[b]=function(a,b,d){var e,f,g=b.toLowerCase();r [...] diff --git a/www/moderation/desk/public/assets/vue.min.js b/www/moderation/desk/public/assets/vue.min.js new file mode 100644 index 0000000..836793b --- /dev/null +++ b/www/moderation/desk/public/assets/vue.min.js @@ -0,0 +1,6 @@ +/*! + * Vue.js v2.5.13 + * (c) 2014-2017 Evan You + * Released under the MIT License. + */ +!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?module.exports=e():"function"==typeof define&&define.amd?define(e):t.Vue=e()}(this,function(){"use strict";function t(t){return void 0===t||null===t}function e(t){return void 0!==t&&null!==t}function n(t){return!0===t}function r(t){return"string"==typeof t||"number"==typeof t||"symbol"==typeof t||"boolean"==typeof t}function i(t){return null!==t&&"object"==typeof t}function o(t){return"[object Object]"===Nn.call(t)}funct [...] \ No newline at end of file diff --git a/www/moderation/desk/public/secmail.css b/www/moderation/desk/public/secmail.css new file mode 100644 index 0000000..344d537 --- /dev/null +++ b/www/moderation/desk/public/secmail.css @@ -0,0 +1,114 @@ +header h1, header h3 { + margin-top: 0; +} + +#messages table { + margin-left: 8px; +} + +#messages td { + padding-right: 7px; + padding-left: 7px; +} + +#messages button { + margin: 5px; +} + +.similar td { + padding-right: 7px; +} + +.selected { + background-color: yellow; +} + +.deleted { + opacity: 0.5; +} + +.doctype label { + display: block +} + +.doctype input[type=radio] { + width: 3em +} + +#parts h4 { + padding: 10px 1em; + background-color: #d9e9f7; +} + +table.form { + width: 100%; + margin-left: 1em; + margin-top: 2em; +} + +.form input, .form textarea, .form select { + width: 100%; +} + +.form input:invalid, .form select:invalid, .form textarea:invalid { + border: 1px solid red; +} + +.form th { + max-width: 4em; + height: 2.5em; + font-weight: normal; +} + +form .btn { + display: block; + margin: auto; + margin-top: 0.5em; +} + +#attachments li.attachment { + width: 100% +} + +div.buttons { + text-align: center; + margin: 1em 0; +} + +ul#editPart { + padding: 0 1.5em; +} + +#editPart li { + list-style: none; + padding: 8; + margin: 16px 0; +} + +.busy, .busy input, .busy input:disabled, .busy .contextMenu li:hover { + cursor: wait; +} + +.divider { + height: 2px; + width:100%; + margin: 9px 0; + padding: 0 4px; + background-color: #ccc; + } + +.spinner { + position: absolute; + z-index: -1; + right: 0; + top: 0; +} + +pre.bg-info { + white-space: pre-wrap; + word-break: normal; +} + +#parts table { + margin-top: 14px; +} diff --git a/www/moderation/desk/public/transparent.png b/www/moderation/desk/public/transparent.png new file mode 100644 index 0000000..6a69b19 Binary files /dev/null and b/www/moderation/desk/public/transparent.png differ diff --git a/www/moderation/desk/server.rb b/www/moderation/desk/server.rb new file mode 100644 index 0000000..db0e3f6 --- /dev/null +++ b/www/moderation/desk/server.rb @@ -0,0 +1,198 @@ +# +# Simple web server that routes requests to views based on URLs. +# + +require 'wunderbar/sinatra' +require 'wunderbar/bootstrap' +require 'wunderbar/vue' +require 'ruby2js/es2017/strict' +require 'ruby2js/filter/functions' +require 'ruby2js/filter/require' +#require 'erb' +#require 'sanitize' +require 'escape' + +require_relative 'helpers' +require_relative 'models/mailbox' +require_relative 'models/safetemp' +require_relative 'models/events' + + +# monkey patch mail gem to work around a regression introduced in 2.7.0: +# https://github.com/mikel/mail/pull/1168 +module Mail + class Message + def raw_source=(value) + @raw_source = ::Mail::Utilities.to_crlf(value) + end + end + + module Utilities + def self.safe_for_line_ending_conversion?(string) + if RUBY_VERSION >= '1.9' + string.ascii_only? or + (string.encoding != Encoding::BINARY and string.valid_encoding?) + else + string.ascii_only? + end + end + end +end + +require 'whimsy/asf' +ASF::Mail.configure + +set :show_exceptions, true + +disable :logging # suppress log of requests to stderr/error.log + +get '/' do + # Ensure trailing slash is present + redirect to('/') if env['REQUEST_URI'] == env['SCRIPT_NAME'] + _html :main +end + +# initial list of messages +get '/messages' do # must agree with src in main.html + + @mbox = Mailbox.mboxname() + @messages = Mailbox.new(@mbox).client_headers + + @cssmtime = File.mtime('public/secmail.css').to_i + @appmtime = Wunderbar::Asset.convert(File.join(settings.views, 'app.js.rb')).mtime.to_i + _html :messages # must agree with views/*.html.rb +end + +# alias for root directory +get '/index.html' do + call env.merge('PATH_INFO' => '/') +end + +# initial list of messages +get %r{/(#{MBOX_RE})/messages} do |mbox| + + @mbox = mbox + @messages = Mailbox.new(@mbox).client_headers + + @cssmtime = File.mtime('public/secmail.css').to_i + @appmtime = Wunderbar::Asset.convert(File.join(settings.views, 'app.js.rb')).mtime.to_i + _html :messages # must agree with views/*.html.rb +end + +# support for fetching next lot of messages +get %r{/(#{MBOX_RE})} do |mbox| + @mbox = mbox + _json Mailbox.new(@mbox).client_headers +end + +# retrieve a single message (same as body now) +get %r{/(#{MBOX_RE})/(#{HASH_RE})/} do |mbox, hash| + @message = Mailbox.new(mbox).find(hash) + return [404, {}, 'Message not found or is not accessible'] unless @message + @attachments = @message.attachments + @headers = @message.headers.dup + @headers.delete :attachments + @cssmtime = File.mtime('public/secmail.css').to_i + _html :body #:message # must agree with views/*.html.rb +end + +# posted actions +post '/actions/:file' do + _json :"actions/#{params[:file]}" +end + +# update a single message status (:Accept, :Reject, :Spam etc) +patch %r{/(#{MBOX_RE})/(#{HASH_RE})/} do |mbox, hash| + + updates = JSON.parse(request.env['rack.input'].read) + + success = Mailbox.patch!(mbox, hash, updates) + + return [404, {}, 'Message not found or is not accessible or could not be updated'] unless success + + [204, {}, ''] +end + +# message body for a single message +get %r{/(#{MBOX_RE})/(#{HASH_RE})/_body_} do |mbox, hash| + @message = Mailbox.new(mbox).find(hash) + return [404, {}, 'Message not found or is not accessible'] unless @message + @attachments = @message.attachments + @headers = @message.headers.dup + @headers.delete :attachments + @cssmtime = File.mtime('public/secmail.css').to_i + _html :body # uses view/body.html.rb +end + +# header data for a single message +get %r{/(#{MBOX_RE})/(#{HASH_RE})/_headers_} do |mbox, hash| + @headers = Mailbox.new(mbox).headers(hash) + return [404, {}, 'Message not found or is not accessible'] unless @headers + _html :headers # uses view/headers.html.rb +end + +# raw data for a single message +get %r{/(#{MBOX_RE})/(#{HASH_RE})/_raw_} do |mbox, hash| + message = Mailbox.new(mbox).find(hash) + return [404, {}, 'Message not found or is not accessible'] unless message + [200, {'Content-Type' => 'text/plain'}, message.raw] +end + +# original data for a single message (testing) +get %r{/(#{MBOX_RE})/(#{HASH_RE})/_orig_} do |mbox, hash| + message = Mailbox.new(mbox).orig(hash) + return [404, {}, 'Message not found or is not accessible'] unless message + [200, {'Content-Type' => 'text/plain'}, message] +end + +# intercede for potentially dangerous message attachments +get %r{/(#{MBOX_RE})/(#{HASH_RE})/_danger_/(.*?)} do |mbox, hash, name| + message = Mailbox.new(mbox).find(hash) + return [404, {}, 'Message not found or is not accessible'] unless message + + @part = message.find(URI.decode(name)) + return [404, {}, 'Attachment not found'] unless @part + + _html :danger +end + +# a specific attachment for a message (e.g. CID) +# WARNING catches anything not handled above! +get %r{/(#{MBOX_RE})/(#{HASH_RE})/(.*?)} do |mbox, hash, name| + message = Mailbox.new(mbox).find(hash) + return [404, {}, 'Message not found or is not accessible'] unless message + + part = message.find(URI.decode(name)) + return [404, {}, 'Attachment not found'] unless part + + [200, {'Content-Type' => part.content_type}, part.body.to_s] +end + +# event stream for server sent events (a.k.a EventSource) +get '/events', provides: 'text/event-stream' do + events = Events.new + + stream :keep_open do |out| + out.callback {events.close} + + loop do + event = events.pop + + if Hash === event or Array === event + out << "data: #{JSON.dump(event)}\n\n" + elsif event == :heartbeat + out << ":\n" + elsif event == :exit + out.close + break + else + out << "data: #{event.inspect}\n\n" + end + end + end +end + +# catch everything else +get %r{/(.+)} do |req| + [500, {}, "I don't understand the request: #{req}"] +end diff --git a/www/moderation/desk/templates/reject-off-topic.erb b/www/moderation/desk/templates/reject-off-topic.erb new file mode 100644 index 0000000..082777e --- /dev/null +++ b/www/moderation/desk/templates/reject-off-topic.erb @@ -0,0 +1,7 @@ +%%% Start comment +Sorry, but the email is off-topic for this mailing list (<%= @mailing_list %>) + +Please etc. + +<%= @sig %> +%%% End comment \ No newline at end of file diff --git a/www/moderation/desk/views/actions/email.json.rb b/www/moderation/desk/views/actions/email.json.rb new file mode 100644 index 0000000..dbc16bc --- /dev/null +++ b/www/moderation/desk/views/actions/email.json.rb @@ -0,0 +1,72 @@ +require 'erb' + +require_relative '../../defines' + +# TODO method this belongs elsewhere +def template(name) + path = File.expand_path("../../../templates/#{name}.erb", __FILE__.untaint) + ERB.new(File.read(path.untaint).untaint).result(binding) +end + + +# extract message headers +headers = Mailbox.hdrs(@id) + +return {error: 'not found', id: @id} unless headers + +allow = headers[:allow] +accept = headers[:accept] +reject = headers[:reject] + +body = '' +case @action + # These values must agree with messages.js.rb + when ':Accept' + to = accept + when ':AcceptAllow' + to = [accept, allow] + when ':Reject' + to = reject + body = template('reject-off-topic') # TODO + else + return {error: 'action missing or unknown', action: @action} +end +# +# obtain per-user information +user = env.user +person = ASF::Person.find(user) + +from = "#{person.public_name} <#{user}@apache.org>".untaint + +# +######################################################################### +## build email # +######################################################################### +# +# build new message +mail = Mail.new +mail.subject = "#{@action} #{@id}" +mail.to = to +mail.from = from + +mail.text_part = body + +# # deliver mail +# complete do +# mail.deliver! +# end + +{ + id: @id, + action: @action, +# headers: headers.inspect, +# allow: allow, +# accept: accept, +# reject: reject, +# from: from, +# [:@_scope, :@_target, :@file, :@selected, :@name, :@default_layout, :@preferred_extension, :@app, :@template_cache, :@request, :@response]" +# vars: self.instance_variables.inspect, +# request: @request.inspect, # Sinatra +# response: @response.inspect, # Sinatra + mail: mail.to_s, +} \ No newline at end of file diff --git a/www/moderation/desk/views/app.js.rb b/www/moderation/desk/views/app.js.rb new file mode 100644 index 0000000..8657fb3 --- /dev/null +++ b/www/moderation/desk/views/app.js.rb @@ -0,0 +1,10 @@ +# This creates the app.js script from all the other *.js.rb files + +require_relative 'vue-config' + +require_relative 'http' +require_relative 'status' + +require_relative 'messages' + +require_relative '../defines' \ No newline at end of file diff --git a/www/moderation/desk/views/body.html.rb b/www/moderation/desk/views/body.html.rb new file mode 100644 index 0000000..c980c59 --- /dev/null +++ b/www/moderation/desk/views/body.html.rb @@ -0,0 +1,124 @@ +# +# View the email content +# + +_html do + _link rel: 'stylesheet', type: 'text/css', + href: "../../secmail.css?#{@cssmtime}" + + # + # Selected headers + # + _table do + if @headers[:list] + _tr do + _td 'Mailing-list:' + _td @headers[:list] + '@' + @headers[:domain] + end + end + _tr do + _td 'From:' + _td @message.from + end + + _tr do + _td 'Return-Path: ' + _td @message.return_path + end + + _tr do + _td 'To:' + _td @message.to + end + + if @message.cc and not @message.cc.empty? + _tr do + _td 'Cc:' + _td @message.cc.join(', ') + end + end + + _tr do + _td 'Subject:' + _td @message.subject || '(empty)' + end + @attachments.each do |att| + attname = att[:name] + attid = att['Content-ID'] + if attname =~ /\.(pdf|txt|jpeg|jpg|gif|png)$/i + link = "#{attid}" + else + link = "_danger_/#{attid}" + end + _tr do + _td 'Attachment: ' + _td do + _a attname, href: link, target: 'content' + end + end + end + end + + _p + + # + # Try various ways to display the body + # + if @message.html_part + _div do # N.B. this is needed for HTML output + container = @message.html_part + body = container.body.to_s + + # Debug + _.comment! "html_part: body.encoding=#{body.encoding} container.charset=#{container.charset}" + + if body.encoding == Encoding::BINARY and container.charset + body.force_encoding(container.charset) rescue nil + end + + nodes = _{body.encode('utf-8', invalid: :replace, :undef => :replace)} + + fixup_images(nodes) + end + elsif @message.text_part + container = @message.text_part + body = container.body.to_s + + # Debug + _.comment! "text_part: body.encoding=#{body.encoding} container.charset=#{container.charset}" + + if body.encoding == Encoding::BINARY and container.charset + body.force_encoding(container.charset) rescue nil + end + + _pre.bg_info body.encode('utf-8', invalid: :replace, :undef => :replace) + else # must be a non-multi part mail + container = @message.mail + body = container.body.to_s + + # Debug + _.comment! "body.encoding=#{body.encoding} container.charset=#{container.charset} container.mime_type=#{container.mime_type}" + + if body.encoding == Encoding::BINARY and container.charset + body.force_encoding(container.charset) rescue nil + end + + if container.mime_type == 'text/plain' + + _pre.bg_info body.encode('utf-8', invalid: :replace, :undef => :replace) + + elsif container.mime_type == 'text/html' + + _div do # N.B. this is needed for HTML output + nodes = _{body.encode('utf-8', invalid: :replace, :undef => :replace)} + + fixup_images(nodes) + end + + else + + _p "(Cannot handle mime-type #{mime_type})" + + end + end +end diff --git a/www/moderation/desk/views/danger.html.rb b/www/moderation/desk/views/danger.html.rb new file mode 100644 index 0000000..3d6006d --- /dev/null +++ b/www/moderation/desk/views/danger.html.rb @@ -0,0 +1,21 @@ +_html do + _h1.bg_danger 'Potentially Dangerous Content' + + _table.table.table_bordered do + _tbody do + @part.headers.each do |name, value| + next if name == :mime + _tr do + _td name.to_s + if name == :name + _td do + _a value, href: "../#{value}" + end + else + _td value + end + end + end + end + end +end diff --git a/www/moderation/desk/views/headers.html.rb b/www/moderation/desk/views/headers.html.rb new file mode 100644 index 0000000..aa499ae --- /dev/null +++ b/www/moderation/desk/views/headers.html.rb @@ -0,0 +1,7 @@ +# +# Dump headers +# + +_html do + _pre YAML.dump(@headers) +end diff --git a/www/moderation/desk/views/http.js.rb b/www/moderation/desk/views/http.js.rb new file mode 100644 index 0000000..7fd7c06 --- /dev/null +++ b/www/moderation/desk/views/http.js.rb @@ -0,0 +1,135 @@ +# +# Encapsulations for asynchronous HTTP requests. Uses older XMLHttpRequest +# API over fetch as fetch isn't widely supported yet: +# https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API#Browser_compatibility +# + + +class HTTP + # "AJAX" style post request to the server, with a callback + def self.post(target, data) + return Promise.new do |resolve, reject| + xhr = XMLHttpRequest.new() + xhr.open('POST', target, true) + xhr.setRequestHeader('Content-Type', 'application/json;charset=utf-8') + xhr.responseType = 'text' + + def xhr.onreadystatechange() + if xhr.readyState == 4 + begin + if xhr.status == 200 + data = JSON.parse(xhr.responseText) + if data.exception + reject(data.exception) + else + resolve(data) + end + else + HTTP._reject(xhr, reject) + end + rescue => e + reject(e) + end + end + end + + xhr.send(JSON.stringify(data)) + end + end + + # "AJAX" style patch request to the server, with a callback + def self.patch(target, data) + return Promise.new do |resolve, reject| + xhr = XMLHttpRequest.new() + xhr.open('PATCH', target, true) + xhr.setRequestHeader('Content-Type', 'application/json;charset=utf-8') + + def xhr.onreadystatechange() + if xhr.readyState == 4 + begin + if xhr.status == 200 + data = JSON.parse(xhr.responseText) + if data.exception + reject(data.exception) + else + resolve(data) + end + elsif xhr.status == 204 + resolve() + else + HTTP._reject(xhr, reject) + end + rescue => e + reject(e) + end + end + end + + xhr.send(JSON.stringify(data)) + end + end + + # "AJAX" style get request to the server, with a callback + def self.get(target, type) + console.log 'GET ' + target + ' ' + type + return Promise.new do |resolve, reject| + xhr = XMLHttpRequest.new() + + def xhr.onreadystatechange() + if xhr.readyState == 4 + begin + if xhr.status == 200 + if type == :json + data = xhr.response || JSON.parse(xhr.responseText) + else + data = xhr.responseText + end + + resolve data + else + HTTP._reject(xhr, reject) + end + rescue => e + reject e + end + end + end + + if target =~ /^https?:/ + xhr.open('GET', target, true) + xhr.setRequestHeader("Accept", "application/json") if type == :json + else + xhr.open('GET', target, true) + end + + xhr.responseType = type + xhr.send() + end + end + + # common rejection logic + def self._reject(xhr, reject) + if not xhr.status + reject "Server unavailable" + elsif xhr.status == 404 + reject "Not found" + else + console.log xhr.response + if not xhr.response + reject "Exception - #{xhr.statusText}" + elsif xhr.response.exception + reject "Exception\n#{xhr.response.exception}" + else + text = xhr.responseText + begin + json = JSON.parse(text) + text = "Exception: #{json.exception}" if json.exception + rescue => e + end + reject text + end + end + rescue => e + reject e + end +end diff --git a/www/moderation/desk/views/main.html.rb b/www/moderation/desk/views/main.html.rb new file mode 100644 index 0000000..64464f1 --- /dev/null +++ b/www/moderation/desk/views/main.html.rb @@ -0,0 +1,14 @@ +# +# Layout for main page +# LH: list of messages +# RH: detail of selected message +# + +_html do + _title 'ASF Moderation Helper' + + _frameset cols: '45%, *' do + _frame src: 'messages' # list of clickable messages must agree with server.rb + _frame name: 'content' # name must agree with link target in message.js.rb + end +end diff --git a/www/moderation/desk/views/messages.html.rb b/www/moderation/desk/views/messages.html.rb new file mode 100644 index 0000000..d2c94a3 --- /dev/null +++ b/www/moderation/desk/views/messages.html.rb @@ -0,0 +1,27 @@ +# Listing of messages + +_html do + if ENV["RACK_BASE_URI"].to_s + '/' == _.env['REQUEST_URI'] + # not sure why Passenger/rack is eating the trailing slash here. + # add it back in. + _base href: _.env['REQUEST_URI'] + end + + _title 'ASF Moderation Helper' + _link rel: 'stylesheet', type: 'text/css', href: "secmail.css?#{@cssmtime}" + + _header_ do + _h1.bg_success do + _a 'ASF Moderation Helper', href: '.', target: '_top' + end + end +# _ __FILE__ + + _div_.messages! # must agree with below (and CSS) + + _script src: "./app.js?#{@appmtime}" + _.render '#messages' do # must agree with above + _Messages mbox: @mbox, messages: @messages + end + +end diff --git a/www/moderation/desk/views/messages.js.rb b/www/moderation/desk/views/messages.js.rb new file mode 100644 index 0000000..bfb8cc9 --- /dev/null +++ b/www/moderation/desk/views/messages.js.rb @@ -0,0 +1,321 @@ +# +# List page showing messages +# + +class Messages < Vue + def initialize + console.log "initialise" + @selected = nil + @messages = [] + @checking = false + @fetched = false + @nextmbox = nil + end + + def render + console.log "render" + if not @messages + _p.container_fluid 'All documents have been processed.' + else + _p "Count: #{@messages.length}" + + # The name values must agree with action scripts such as email.json.rb + _button.btn.btn_info 'Accept and Allow', onClick: self.mark, name: ACCEPTALLOW, disabled: !@selected + _button.btn.btn_info 'Accept only', onClick: self.mark, name: ACCEPT, disabled: !@selected + _button.btn.btn_info 'Reject', onClick: self.mark, name: REJECT, disabled: !@selected + _button.btn.btn_info 'This is Spam', onClick: self.mark, name: MARKSPAM, disabled: !@selected + _button.btn.btn_info 'Unmark Spam', onClick: self.undo, disabled: Status.undoStack.empty? + # TODO accept subscription request - or just re-use Accept? + # COuld use Reject as well to tell the user why not accepting the request + # Also want to be able to ignore/delete the request + + _table.table do + + _tbody do + @messages.each do |message| + + # determine the 'color' to use for the row + color = nil + color = 'deleted' if message.status == :deletePending + color = 'hidden' if message.status == :deleted # should be temporary until next response + color = 'selected' if message.href == @selected + + row_options = { + :class => color, + on: {click: self.selectRow, doubleClick: self.nav} + } + + _tr row_options do + _td :id => message.href do # Id needed for mouse selection + _a message.subject, href: "#{message.href}_body_", target: 'content' +# _ message.subject + _br + _ message.from + _br + _a message.return_path, href: "#{message.href}_headers_", target: 'content' +# _ message.return_path + end + _td do + _ "#{message.list}@#{message.domain}" + _br + _ Date.new(message.timestamp*1000.to_i).toISOString() if message.timestamp + # href must == message.href otherwise selection does not work + # Note: frame has URL of href + # target must agree with name in main.html.rb + _br + _a message.href, href: "#{message.href}_raw_", target: 'content' + end + end + end + end + end + end + + unless Status.undoStack.empty? + _button.btn.btn_info 'undo delete', onClick: self.undo + end + end + + # initialize; store passed messages + def beforeMount() + Status.emptyStack() + @nextmbox = @@messages.nextmbox if @@messages + self.merge @@messages if @@messages + console.log "beforeMount next #{@nextmbox}" + end + + def fetch_mbox(&block) + console.log "fetch_mbox> #{@nextmbox}" + HTTP.get(@nextmbox, :json).then {|response| + @nextmbox = response.nextmbox + + # add messages to list + self.merge response + + # if block provided, call it + block() if block and block.is_a? Function + }.catch {|error| + console.log error + alert error + } + console.log "fetch_mbox< #{@nextmbox}" + end + + def handle_response() + console.log "handle_response next #{@nextmbox} max #{@max_fetch}" + @max_fetch -= 1 + if @nextmbox and @max_fetch > 0 + fetch_mbox() do handle_response() end + else + # select oldest message + self.selectRow Status.selected || @messages.first + end + end + + # on initial load, subscribe to keyboard and + # server side events, and initialize selected item. + def mounted() + console.log "mounted next #{@nextmbox}" + @max_fetch = 15 # prevent excess fetches + fetch_mbox() do handle_response() end if @nextmbox + + window.onkeydown = self.keydown + + # when events are received, update messages + events = EventSource.new('events') + events.addEventListener :message do |event| + messages = JSON.parse(event.data).messages + if messages + console.log "Message event, source: #{messages.source} count: #{messages.headers.length}" + else + console.log event + end + self.merge messages if messages + end + + # close connection on exit + window.addEventListener :unload do |event| + events.close() + end + + # select row + console.log "mounted selected #{Status.selected}" + self.selectRow Status.selected if @messages.length > 0 + end + + # when content changes, ensure selected message is visible + def updated() + if @selected + selected = document.querySelector("td[id='#{@selected}']") + if selected + rect = selected.getBoundingClientRect() + if + rect.top < 0 or rect.left < 0 or + rect.bottom > window.innerHeight or rect.right > window.innerWidth + then + selected.scrollIntoView() + end + end + end + end + + # merge new messages into the list + def merge(messages) + source = messages.source + headers = messages.headers + console.log "merge: " + source + " Count: " + headers.length + # Drop all entries from the same source file + temp = @messages.select { |k| not k.href.start_with? source+"/"} + headers.each do |hdr| + hdr[:href] = source + "/" + hdr[:id] + "/" # construct the href + hdr[:id].delete # no longer needed + # Where to insert the new message (ascending order - oldest first as they may expire) + index = temp.find_index do |old| + old.timestamp > hdr.timestamp + end + if index == -1 # not found, i.e. new time is > all entries, so add at end + temp << hdr + else # found a newer entry at index, want to insert ahead of it + temp.splice index, 0, hdr + end + end + @messages = temp + Vue.forceUpdate() unless messages.empty? + end + + # update @selected, given either a DOM event or a message + def selectRow(object) + hasLink = nil # did we click a link? +# console.log "SelectRow #{object} #{typeof(object)}" + if not object +# console.log "A" + href = nil + elsif typeof(object) == 'string' +# console.log "B" + href = object + elsif object.respond_to? :currentTarget +# console.log "C #{object.srcElement.href}" + hasLink = object.srcElement.href + href = object.currentTarget.querySelector('td').getAttribute('id') + elsif object.respond_to? :href +# console.log "D" + href = object.href + else +# console.log "E" + href = object + end + + # ensure selected message is not deleted + index = @messages.find_index {|m| m.href == href} + index -= 1 while index >= 0 and @messages[index].status == :deleted + # else find first non-deleted entry + index = @messages.find_index {|m| m.status != :deleted} if index == -1 + + previous = @selected + @selected = Status.selected = (index >= 0 ? @messages[index].href : nil) +# console.log "SelectRow href #{href} index #{index} previous #{previous} selected #{@selected} S.s #{Status.selected}" + if @selected # display the message details + # don't try to display if we have just clicked a link + parent.content.location=@selected unless hasLink + else +# parent.message.document.body.textContent='' TODO + end + end + + # navigate + def nav(event) + self.selectRow(event) + window.location.href = @selected + window.getSelection().removeAllRanges() + event.preventDefault() + end + + def send_email(data, &block) + console.log "send_email > #{data.inspect}" + HTTP.post('actions/email', data).then {|response| + console.log "send_email < #{response.inspect}" + alert response[:mail] + block() if block + }.catch {|error| + alert error + } + end + + def mark(event) + name=event.srcElement.name + selected = @selected + if selected + event.preventDefault() + # mark item as delete pending + index = @messages.find_index {|m| m.href == selected} + @messages[index].status = :deleted if index >= 0 + # move selected pointer to next message + if index >= 0 and index < @messages.length - 1 + self.selectRow @messages[index+1] + elsif index > 0 and index == @messages.length - 1 + self.selectRow @messages[index-1] + else + self.selectRow nil + end + + unless name == MARKSPAM + send_email({id: selected, action: name}) { + alert "Now patch" + } + else + # TEMP don't delete + HTTP.patch(selected, status: name).then { + @messages[index].status = :deleted if index >= 0 + Status.pushDeleted selected if name == MARKSPAM + self.selectRow @selected # selected above + Vue.forceUpdate() + }.catch {|error| + alert error + } + end + else + alert "Please select a row" + end + end + + def undo(event) + message = Status.popStack() + selected = @messages.find {|m| m.href == message} + if selected + selected.status = :deletePending + end + # send request to server to remove delete status + HTTP.patch(message, status: nil).then { + Vue.forceUpdate() + self.selectRow message + }.catch {|error| + alert error + } + end + + # handle keyboard events + def keydown(event) + if event.keyCode == 38 # up + index = @messages.find_index {|m| m.href == @selected} + self.selectRow @messages[index-1] if index > 0 + event.preventDefault() + + elsif event.keyCode == 40 # down + index = @messages.find_index {|m| m.href == @selected} + 1 + while index < @messages.length and @messages[index].status == :deleted + index += 1 + end + self.selectRow @messages[index] if index < @messages.length + event.preventDefault() + + elsif event.keyCode == 'Z'.ord + if event.ctrlKey or event.metaKey + unless Status.undoStack.empty? + self.undo() + event.preventDefault() + end + end + else + end + end +end diff --git a/www/moderation/desk/views/status.js.rb b/www/moderation/desk/views/status.js.rb new file mode 100644 index 0000000..d856a9b --- /dev/null +++ b/www/moderation/desk/views/status.js.rb @@ -0,0 +1,53 @@ +# +# Encapsulate memory of selected item and delete stack +# + + +STORAGE_NAME = 'modmail' +class Status + def self.modmail + return {} if not defined? sessionStorage + JSON.parse(sessionStorage.getItem(STORAGE_NAME) || '{}') + end + + def self.undoStack + modmail = Status.modmail + return modmail.undoStack || [] + end + + def self.selected + Status.modmail.selected + end + + def self.selected=(value) +# console.log "Status set selected: #{value}" + modmail = Status.modmail + modmail.selected=value + sessionStorage.setItem(STORAGE_NAME, JSON.stringify(modmail)) + end + + def self.pushDeleted(value) +# console.log "pushDeleted #{value}" + value = value[/\w+\/\w+\/?$/].sub(/\/?$/, '/') + modmail = Status.modmail + modmail.undoStack ||= [] + modmail.undoStack << value + sessionStorage.setItem(STORAGE_NAME, JSON.stringify(modmail)) + end + + def self.popStack() + modmail = Status.modmail + modmail.undoStack ||= [] + item = modmail.undoStack.pop() + sessionStorage.setItem(STORAGE_NAME, JSON.stringify(modmail)) +# console.log "popStack: #{item}" + return item + end + + def self.emptyStack() + modmail = Status.modmail + modmail.undoStack = [] + sessionStorage.setItem(STORAGE_NAME, JSON.stringify(modmail)) + end + +end diff --git a/www/moderation/desk/views/vue-config.js.rb b/www/moderation/desk/views/vue-config.js.rb new file mode 100644 index 0000000..b2e9ae5 --- /dev/null +++ b/www/moderation/desk/views/vue-config.js.rb @@ -0,0 +1,10 @@ +# Filter out "data property already declared as a prop" warnings +Vue.config.warnHandler = proc do |msg, vm, trace| + return if msg =~ /^The data property "\w+" is already declared as a prop\./ + console.error "[Vue warn]: " + msg + trace if defined? console +end + +# reraise uncapturable errors asynchronously to enable easier debugging +Vue.config.errorHandler = proc do |err, vm, info| + setTimeout(0) { raise err } +end
