This is an automated email from the ASF dual-hosted git repository.
curcuru pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/whimsy.git
The following commit(s) were added to refs/heads/master by this push:
new b3499e0 Refactor and add some mbox features
b3499e0 is described below
commit b3499e09b7519dec3d5bf9c99dbd9d48c7853693
Author: Shane Curcuru <[email protected]>
AuthorDate: Mon Dec 11 19:52:03 2017 -0500
Refactor and add some mbox features
---
tools/ponyapi.rb | 88 ++++++++++++++++++++++++++++++++++++++++++++
tools/ponypoop.rb | 108 +++++++++++-------------------------------------------
2 files changed, 109 insertions(+), 87 deletions(-)
diff --git a/tools/ponyapi.rb b/tools/ponyapi.rb
new file mode 100644
index 0000000..c8be57d
--- /dev/null
+++ b/tools/ponyapi.rb
@@ -0,0 +1,88 @@
+#!/usr/bin/env ruby
+<<~HEREDOC
+Pony down: utilities for downloading Ponymail APIs (stats.lua or mbox.lua)
+See also: https://ponymail.incubator.apache.org/docs/api
+HEREDOC
+require 'json'
+require 'csv'
+require 'net/http'
+require 'cgi'
+
+# Utilities for downloading from Ponymail APIs
+module PonyAPI
+ PONYSTATS = 'https://lists.apache.org/api/stats.lua?list=' #
board&domain=apache.org&d=2017-04 becomes board-apache-org-201704-stats.json
+ PONYMBOX = 'https://lists.apache.org/api/mbox.lua?list=' #
[email protected]&date=2016-06 becomes board-apache-org-201707.mbox
+
+ extend self
+
+ # Fetch a Ponymail API, with optional logged-in cookie
+ def fetch_pony(uri, cookie)
+ uri = URI.parse(uri)
+ Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |https|
+ request = Net::HTTP::Get.new(uri.request_uri)
+ request['Cookie'] = "ponymail=#{cookie}" if cookie != ''
+ response = https.request(request)
+ if response.code =~ /^3\d\d/
+ fetch_pony response['location'], cookie
+ else
+ return uri, request, response
+ end
+ end
+ end
+
+ # Download one month of stats as a JSON
+ # Must supply cookie = 'ponymail-logged-in-cookie' if a private list
+ def get_pony_stats(dir, list, subdomain, year, month, cookie)
+ if subdomain.nil? || subdomain == ''
+ getlist = "#{list}&domain=apache.org"
+ fname = "#{list}-apache-org-#{year}#{month}-stats.json"
+ else
+ getlist = "#{list}&domain=#{subdomain}.apache.org"
+ fname = "#{list}-#{subdomain}-apache-org-#{year}#{month}-stats.json"
+ end
+ uri, request, response =
fetch_pony("#{PONYSTATS}#{getlist}&d=#{year}-#{month}", cookie)
+ if response.code =~ /200/ then
+ File.open(File.join("#{dir}", "#{fname}"), "w") do |f|
+ jzon = JSON.parse(response.body)
+ begin
+ f.puts JSON.pretty_generate(jzon)
+ rescue JSON::GeneratorError
+ puts "WARN:get_pony_stats(#{uri.request_uri}) threw
JSON::GeneratorError, continuing without pretty"
+ f.puts jzon
+ end
+ end
+ else
+ puts "ERROR:get_pony_stats(#{uri.request_uri}) returned code
#{response.code}"
+ end
+ end
+
+ # Get multiple years/months of public stats as json
+ def get_pony_stats_many(dir, list, subdomain, years, months, cookie)
+ years.each do |y|
+ months.each do |m|
+ get_pony_stats dir, list, subdomain, y, m, cookie
+ end
+ end
+ end
+
+ # Download one month as mbox
+ # Caveats: uses response's encoding; overwrites existing .json file
+ # Must supply cookie = 'ponymail-logged-in-cookie' if a private list
+ def get_pony_mbox(dir, list, subdomain, year, month, cookie)
+ if subdomain.nil? || subdomain == ''
+ getlist = "#{list}@apache.org"
+ fname = "#{list}-apache-org-#{year}#{month}.mbox"
+ else
+ getlist = "#{list}@#{subdomain}.apache.org"
+ fname = "#{list}-#{subdomain}-apache-org-#{year}#{month}.mbox"
+ end
+ uri, request, response =
fetch_pony("#{PONYMBOX}#{getlist}&date=#{year}-#{month}", cookie)
+ if response.code =~ /^200/
+ File.open(File.join("#{dir}", "#{fname}"),
"w:#{response.body.encoding}") do |f|
+ f.puts response.body
+ end
+ else
+ puts "ERROR:get_public_mbox(#{uri}) returned code #{response.code}"
+ end
+ end
+end
\ No newline at end of file
diff --git a/tools/ponypoop.rb b/tools/ponypoop.rb
index 7e96cbc..f9cbcb3 100755
--- a/tools/ponypoop.rb
+++ b/tools/ponypoop.rb
@@ -1,23 +1,22 @@
#!/usr/bin/env ruby
<<~HEREDOC
-Pony poop: simple statistics for Apache Ponymail monthly archives
- - Methods to pull down stats.lua JSON structures of monthly archive reports
- - Medhods to analyze local .json structures with chartable stats
-
- See also: https://ponymail.incubator.apache.org/docs/api
- See also: https://lists.apache.org/ngrams.html
+Pony poop: utilities for analyzing data from Apache Ponymail APIs
+- Analyze stats.lua JSON output for subject/author analysis
+- Analyze mbox.lua mbox files for author/list/lines written analysis
+
+See also: https://ponymail.incubator.apache.org/docs/api
+See also: https://lists.apache.org/ngrams.html
HEREDOC
require 'json'
require 'csv'
require 'net/http'
require 'cgi'
require 'optparse'
-
-PONYSTATS = 'https://lists.apache.org/api/stats.lua?list=' #
board&domain=apache.org&d=2017-04 becomes board-apache-org-201704.json
+require_relative 'ponyapi'
# TODO: Fixup CSV output format to be more flexible, and/or add charting
automatically
CSV_COLS = %w( Date TotalEmails TotalInteresting TotalThreads Missing Feedback
Notice Report Resolution SVNAgenda SVNICLAs Person1 Emails1 Person2 Emails2
Person3 Emails3 Person4 Emails4 Person5 Emails5 )
-BOARD_REGEX = { # Non-interesting email subjects from board
+BOARD_REGEX = { # Non-interesting email subjects from board # TODO add
features for other lists
missing: /\AMissing\s\S+\sBoard/,
feedback: /\ABoard\sfeedback\son\s20/,
notice: /\A\[NOTICE\]/i,
@@ -49,7 +48,7 @@ def analyze_threads(threads)
end
# Analyze a local .json for interesting vs. not interesting board@ subjects
-def analyze(fname, results, subject_regex, errors)
+def analyze_stats(fname, results, subject_regex, errors)
begin
f = File.basename(fname)
begin
@@ -105,13 +104,13 @@ def analyze(fname, results, subject_regex, errors)
end
# Analyze a set of local .json files downloaded from lists.a.o
-def run_analyze(dir, list, subject_regex)
+def run_analyze_stats(dir, list, subject_regex)
results = []
errors = []
subjects = []
output = File.join("#{dir}", "output-#{list}")
Dir[File.join("#{dir}", "#{list}*.json")].each do |fname|
- subjects = analyze(fname, results, subject_regex, errors)
+ subjects = analyze_stats(fname, results, subject_regex, errors)
if subjects
responses = subjects.select {|subj| subj =~ /Re:/i }.size
File.open("#{fname.chomp('.json')}.txt", "w") do |f|
@@ -136,77 +135,16 @@ def run_analyze(dir, list, subject_regex)
File.open("#{output}.json", "w") do |f|
f.puts JSON.pretty_generate(results)
end
-
- results
-end
-
-# ## ### #### ##### ######
-# Download functions: grab monthly stats.lua data as .jsons
-# Grab monthly data from lists.a.o - for private lists
-def get_private_from_archive(dir, list, years, months, cookie)
- cookieval = "ponymail=#{cookie}"
- years.each do |y|
- months.each do |m|
- uri = URI("#{PONYSTATS}#{list}&domain=apache-org&d=#{y}-#{m}")
- http = Net::HTTP.new(uri.host, uri.port)
- http.use_ssl = true
- request = Net::HTTP::Get.new(uri.request_uri)
- request['Cookie'] = cookieval
- r = http.request(request)
- if r.code =~ /200/ then
- File.open(File.join("#{dir}", "#{list}-apache-org-#{y}#{m}.json"),
"w") do |f|
- jzon = JSON.parse(r.body)
- begin
- f.puts JSON.pretty_generate(jzon)
- rescue JSON::GeneratorError
- puts "Bogosity: Generator error on #{r.code} for
#{uri.request_uri}"
- f.puts jzon
- end
- end
- else
- puts "Double Bogus! #{r.code} for #{uri.request_uri}"
- end
- end
- end
-end
-
-# ## ### #### ##### ######
-# Grab monthly data from lists.a.o - only for public lists
-# fetch uri, following redirects: tools/site-scan.rb
-def fetch(uri)
- uri = URI.parse(uri)
- Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
- request = Net::HTTP::Get.new(uri.request_uri)
- response = http.request(request)
- if response.code =~ /^3\d\d/
- fetch response['location']
- else
- return uri, request, response
- end
- end
-end
-
-# Grab monthly data from lists.a.o - for public lists
-def get_public_from_archive(dir, list, subdomain, year, month)
- uri, request, response =
fetch("#{PONYSTATS}#{list}&domain=#{subdomain}.apache.org&d=#{year}-#{month}")
- pmails = JSON.parse(response.body)
- File.open(File.join("#{dir}",
"#{list}-#{subdomain}-apache-org-#{year}-#{month}.json"), "w") do |f|
- f.puts JSON.pretty_generate(pmails)
- end
+ results
end
-def get_all_public(dir, list, subdomain, years, months)
- years.each do |y|
- months.each do |m|
- get_public_from_archive dir, list, subdomain, y, m
- end
- end
-end
# ## ### #### ##### ######
# Check options and call needed methods
-# TODO: this assumes you correctly use -c and -s
+# TODO: Simplify and allow both:
+# - Downloading either stats or mbox
+# - Analyzing either stats or mbox
def optparse
options = {}
OptionParser.new do |opts|
@@ -223,14 +161,14 @@ def optparse
options[:list] = l.chomp('@')
end
- opts.on('-cCOOKIE', '--cookie COOKIE', 'For private lists, your ponymail
logged-in cookie value') do |c|
+ opts.on('-cCOOKIE', '--cookie COOKIE', 'For private lists REQUIRED, your
ponymail logged-in cookie value') do |c|
options[:cookie] = c
end
- opts.on('-sSUBDOMAIN', '--list SUBDOMAIN', 'Root @ subdomain .apache.org
(only if project list; hadoop or community or...) to download stats archive
from') do |s|
+ opts.on('-sSUBDOMAIN', '--subdomain SUBDOMAIN', 'Root @ subdomain
.apache.org (only if project list; hadoop or community or...) to download stats
archive from') do |s|
options[:subdomain] = s.chomp('@.')
end
- opts.on('-p', '--pull', 'Pull down stats into -d dir (otherwise, analyzes
existing stats in dir)') do |p|
+ opts.on('-p', '--pull', 'Pull down stats into -d dir (otherwise, default
analyzes existing stats in dir)') do |p|
options[:pull] = true
end
@@ -254,16 +192,12 @@ if __FILE__ == $PROGRAM_NAME
options = optparse
options[:list] ||= 'board'
if options[:pull]
- # TODO make months/years settable
- raise ArgumentError "Must have a -c COOKIE to -p pull private archives"
unless options[:cookie]
- puts "BEGIN: Pulling down JSON to #{options[:dir]} of list:
#{options[:list]} @ #{options[:subdomain]} "
- get_private_from_archive options[:dir], options[:list], years, months,
options[:cookie]
+ puts "BEGIN: Pulling down stats JSONs in #{options[:dir]} of list:
#{options[:list]}@#{options[:subdomain]}"
+ PonyAPI::get_pony_stats_many options[:dir], options[:list],
options[:subdomain], years, months, options[:cookie]
else
puts "BEGIN: Analyzing local JSONs in #{options[:dir]} of list:
#{options[:list]}"
- run_analyze options[:dir], options[:list], BOARD_REGEX
+ run_analyze_stats options[:dir], options[:list], BOARD_REGEX
end
puts "END: Thanks for running ponypoop - see results in #{options[:dir]}"
end
-
-
--
To stop receiving notification emails like this one, please contact
['"[email protected]" <[email protected]>'].
