This is an automated email from the ASF dual-hosted git repository.
sebb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/whimsy.git
The following commit(s) were added to refs/heads/master by this push:
new 169165a3 Ensure listings are refreshed before use
169165a3 is described below
commit 169165a36bcca0617c609285451adbde603ddb14
Author: Sebb <[email protected]>
AuthorDate: Mon Jan 9 22:21:08 2023 +0000
Ensure listings are refreshed before use
---
lib/whimsy/asf/documents.rb | 60 ++++++++++++++++------
lib/whimsy/asf/memapps.rb | 11 ++--
lib/whimsy/asf/rack.rb | 7 ++-
www/roster/main.rb | 2 +
www/roster/models/committer.rb | 8 +--
www/secretary/emeritus_check.cgi | 12 +++--
www/secretary/icla-dupes.cgi | 4 +-
www/secretary/icla-lint.cgi | 18 ++++---
www/secretary/ldap-check.cgi | 26 +++++-----
www/secretary/ldap-names.cgi | 2 +
www/secretary/memapp_check.cgi | 3 ++
www/secretary/public-names.cgi | 2 +
www/secretary/workbench/server.rb | 1 +
www/secretary/workbench/views/actions/ccla.json.rb | 1 +
.../workbench/views/actions/grant.json.rb | 1 +
.../workbench/views/actions/memapp.json.rb | 1 +
16 files changed, 111 insertions(+), 48 deletions(-)
diff --git a/lib/whimsy/asf/documents.rb b/lib/whimsy/asf/documents.rb
index 4ee9a273..680ff9a5 100644
--- a/lib/whimsy/asf/documents.rb
+++ b/lib/whimsy/asf/documents.rb
@@ -5,8 +5,12 @@ require 'json'
module ASF
module DocumentUtils
- # create/update cache file
- def self.update_cache(type, cache_dir)
+
+ CACHE_DIR = ASF::Config.get(:cache)
+ MAX_AGE = 600 # 5 minutes
+
+ # check cache age and get settings
+ def self.check_cache(type, cache_dir: CACHE_DIR, warn: true)
file, _ = ASF::SVN.listingNames(type, cache_dir)
mtime = begin
File.mtime(file)
@@ -14,10 +18,22 @@ module ASF
0
end
age = (Time.now - mtime).to_i
- if age > 600 # 5 minutes
- Wunderbar.warn "Updating listing #{file} #{age}"
+ stale = age > MAX_AGE
+ if warn && stale
+ Wunderbar.warn "Cache for #{type} is older than #{MAX_AGE} seconds"
+ # Wunderbar.warn caller(0, 10).join("\n")
+ end
+ return [cache_dir, stale, file, age]
+ end
+
+ # create/update cache file
+ def self.update_cache(type, env, cache_dir: CACHE_DIR)
+ _cache_dir, stale, file, age = check_cache(type, cache_dir: cache_dir,
warn: false)
+ if stale
require 'whimsy/asf/rack'
- ASF::Auth.decode(env = {})
+ ASF::Auth.decode(env)
+ # TODO: Downdate to info
+ Wunderbar.warn "Updating listing #{file} #{age} as #{env.user}"
filerev, svnrev = ASF::SVN.updatelisting(type, env.user, env.password,
false, cache_dir)
if filerev && svnrev # it worked
FileUtils.touch file # last time it was checked
@@ -32,12 +48,17 @@ module ASF
# Common class for access to documents/cclas/
class CCLAFiles
+ STEM = 'cclas'
+
+ def self.update_cache(env)
+ ASF::DocumentUtils.update_cache(STEM, env)
+ end
+
# listing of top-level icla file/directory names
# Directories are listed without trailing "/"
def self.listnames
- cache_dir = ASF::Config.get(:cache)
- DocumentUtils.update_cache('cclas', cache_dir)
- _, list = ASF::SVN.getlisting('cclas', nil, true, false, cache_dir)
+ cache_dir = ASF::DocumentUtils.check_cache(STEM).first
+ _, list = ASF::SVN.getlisting(STEM, nil, true, false, cache_dir)
list
end
@@ -51,12 +72,17 @@ module ASF
# Common class for access to documents/cclas/
class GrantFiles
+ STEM = 'grants'
+
+ def self.update_cache(env)
+ ASF::DocumentUtils.update_cache(STEM, env)
+ end
+
# listing of top-level grants file/directory names
# Directories are listed without trailing "/"
def self.listnames
- cache_dir = ASF::Config.get(:cache)
- DocumentUtils.update_cache('grants', cache_dir)
- _, list = ASF::SVN.getlisting('grants', nil, true, false, cache_dir) #
do we need to cache the listing?
+ cache_dir = ASF::DocumentUtils.check_cache(STEM).first
+ _, list = ASF::SVN.getlisting(STEM, nil, true, false, cache_dir)
list
end
@@ -77,6 +103,12 @@ module ASF
@@h_claRef = nil # for matching claRefs
@@h_stem = nil # for matching stems
+ STEM = 'iclas'
+
+ def self.update_cache(env)
+ ASF::DocumentUtils.update_cache(STEM, env)
+ end
+
# search icla files to find match with claRef
# matches if the input matches the full name of a file or directory or
# it matches with an extension
@@ -120,11 +152,9 @@ module ASF
# This returns the list of names in the top-level directory
# directory names are terminated by '/'
def self.listnames
- iclas = 'iclas'
- cache_dir = ASF::Config.get(:cache)
# iclas.txt no longer updated by cronjob
- DocumentUtils.update_cache(iclas, cache_dir)
- @@tag, list = ASF::SVN.getlisting(iclas, @@tag, false, false, cache_dir)
+ cache_dir = ASF::DocumentUtils.check_cache(STEM).first
+ @@tag, list = ASF::SVN.getlisting(STEM, @@tag, false, false, cache_dir)
if list # we have a new list
# update the list cache
@@list = list
diff --git a/lib/whimsy/asf/memapps.rb b/lib/whimsy/asf/memapps.rb
index 0ba16465..8abed036 100644
--- a/lib/whimsy/asf/memapps.rb
+++ b/lib/whimsy/asf/memapps.rb
@@ -12,6 +12,12 @@ module ASF
@@files = nil
@@tag = nil
+ STEM = 'member_apps'
+
+ def self.update_cache(env)
+ ASF::DocumentUtils.update_cache(STEM, env)
+ end
+
# list the stems of the files
def self.stems
refresh
@@ -83,9 +89,8 @@ module ASF
private
def self.refresh
- cache_dir = ASF::Config.get(:cache)
- ASF::DocumentUtils.update_cache('member_apps', cache_dir)
- @@tag, list = ASF::SVN.getlisting('member_apps', @@tag, true, false,
cache_dir)
+ cache_dir = ASF::DocumentUtils.check_cache(STEM).first
+ @@tag, list = ASF::SVN.getlisting(STEM, @@tag, true, false, cache_dir)
if list
@@files = list
end
diff --git a/lib/whimsy/asf/rack.rb b/lib/whimsy/asf/rack.rb
index f4f85729..9027300e 100644
--- a/lib/whimsy/asf/rack.rb
+++ b/lib/whimsy/asf/rack.rb
@@ -10,10 +10,15 @@ module ASF
def self.decode(env)
class << env; attr_accessor :user, :password; end
- auth = env['HTTP_AUTHORIZATION'] || ENV['HTTP_AUTHORIZATION']
+ if ENV['PASSENGER_APP_ENV']
+ auth = env['HTTP_AUTHORIZATION']
+ else # only use ENV if not a Passenger app
+ auth = ENV['HTTP_AUTHORIZATION']
+ end
if auth.to_s.empty?
env.user = env['REMOTE_USER'] || ENV['USER'] || Etc.getpwuid.name
+ Wunderbar.warn "No auth found for #{env.user}"
else
require 'base64'
env.user, env.password = Base64.
diff --git a/www/roster/main.rb b/www/roster/main.rb
index 1186c3fb..31ac919c 100755
--- a/www/roster/main.rb
+++ b/www/roster/main.rb
@@ -195,6 +195,7 @@ get '/committer2/index.json' do
ASF::ICLA.each {|icla|
if icla.noId?
if @auth[:secretary]
+ ASF::ICLAFiles.update_cache(env)
iclaFile = ASF::ICLAFiles.match_claRef(icla.claRef) # must be
secretary
tmp << { name: icla.name, mail: icla.email, claRef: icla.claRef,
iclaFile: iclaFile}
else
@@ -277,6 +278,7 @@ get '/icla/index.json' do
ASF::ICLA.each {|icla|
if icla.noId?
if @auth[:secretary] # only secretary sees ICLAs
+ ASF::ICLAFiles.update_cache(env)
iclaFile = ASF::ICLAFiles.match_claRef(icla.claRef)
icla_index << { name: icla.name, mail: icla.email, claRef:
icla.claRef, iclaFile: iclaFile}
else
diff --git a/www/roster/models/committer.rb b/www/roster/models/committer.rb
index 23283b74..67176e04 100644
--- a/www/roster/models/committer.rb
+++ b/www/roster/models/committer.rb
@@ -131,10 +131,11 @@ class Committer
if auth[:member] # i.e. member karma
- if person.icla and person.icla.claRef and (auth[:secretary] or
auth[:root]) # Not all people have iclas (only check if secretary or root role)
+ if person.icla&.claRef and (auth[:secretary] or auth[:root]) # Not all
people have iclas (only check if secretary or root role)
+ ASF::ICLAFiles.update_cache(env)
file = ASF::ICLAFiles.match_claRef(person.icla.claRef) # must be
secretary or root
if file
- url =ASF::SVN.svnurl('iclas')
+ url = ASF::SVN.svnurl('iclas')
response[:forms][:icla] = "#{url}/#{file}"
end
end
@@ -144,6 +145,7 @@ class Committer
member[:info] = person.members_txt
if person.icla # not all members have iclas
+ ASF::MemApps.update_cache(env)
file = ASF::MemApps.find1st(person)
if file
url = ASF::SVN.svnurl('member_apps')
@@ -182,7 +184,7 @@ class Committer
end
else # not an ASF member; no karma for ICLA docs so don't add link
- response[:forms][:icla] = '' if person.icla and person.icla.claRef
+ response[:forms][:icla] = '' if person.icla&.claRef
end
response[:member] = member unless member.empty?
diff --git a/www/secretary/emeritus_check.cgi b/www/secretary/emeritus_check.cgi
index c37269ee..e7bb268d 100755
--- a/www/secretary/emeritus_check.cgi
+++ b/www/secretary/emeritus_check.cgi
@@ -14,19 +14,21 @@ require 'yaml'
file =
File.join(ASF::SVN.find!('emeritus-involuntary'),'emeritus-involuntary.yml')
forced = Set.new
-YAML.load_file(file).each{|k,v| v.each{|w| forced.add w}}
+YAML.load_file(file).each {|_k, v| v.each {|w| forced.add w}}
exmembers = ASF::Member.emeritus.map {|id| ASF::Person.find(id)}
ASF::Person.preload(['cn'], exmembers) # speed up
files = Hash[ASF::EmeritusFiles::listnames.map{|i| [i,'NAK']}]
nofiles = Hash.new()
+ASF::ICLAFiles.update_cache({})
+
exmembers.each { |m|
ma = ASF::EmeritusFiles.find(m)
if ma
files[ma] = 'OK'
else
- nofiles[m.name]=m
+ nofiles[m.name] = m
end
}
_html do
@@ -46,7 +48,7 @@ _html do
_tr do
_th 'Name'
end
- files.select {|k,v| v == 'NAK'}.sort_by{|k| k[0].split('-').pop}.each do
|k,v|
+ files.select { |_k, v| v == 'NAK'}.sort_by { |k| k[0].split('-').pop}.each
do |k, _v|
_tr do
_td do
_a k, href: ASF::SVN.svnpath!('emeritus', k), target: '_blank'
@@ -64,7 +66,7 @@ _table_ do
_th 'Legal Name'
_th 'Member.txt Name'
end
- nofiles.sort_by{|k,v| v.member_name}.each do |k,v|
+ nofiles.sort_by { |_k, v| v.member_name}.each do |k, v|
person = v
if forced.delete? person.member_name
next
@@ -74,7 +76,7 @@ _table_ do
_a k, href: "https://whimsy.apache.org/roster/committer/#{k}";, target:
'_blank'
end
_td do
- if person.icla && person.icla.claRef
+ if person.icla&.claRef
file = ASF::ICLAFiles.match_claRef(person.icla.claRef)
if file
_a person.icla.claRef, href: ASF::SVN.svnpath!('iclas', file),
target: '_blank'
diff --git a/www/secretary/icla-dupes.cgi b/www/secretary/icla-dupes.cgi
index a1a47f29..edc8c75c 100755
--- a/www/secretary/icla-dupes.cgi
+++ b/www/secretary/icla-dupes.cgi
@@ -8,6 +8,8 @@ require 'wunderbar/script'
require 'ruby2js/filter/functions'
require 'whimsy/asf'
+ASF::ICLAFiles.update_cache({})
+
_html do
_style %{
table {border-collapse: collapse}
@@ -26,7 +28,7 @@ _html do
_p 'Further checks TBA, e.g. looking for partial matches'
- dups = Hash.new{|h,k| h[k]=Array.new}
+ dups = Hash.new { |h, k| h[k] = Array.new }
ASF::ICLA.each do |icla|
legal = icla.legal_name
legals = legal.downcase.split(' ')
diff --git a/www/secretary/icla-lint.cgi b/www/secretary/icla-lint.cgi
index 2d6e3ae8..080b83f8 100755
--- a/www/secretary/icla-lint.cgi
+++ b/www/secretary/icla-lint.cgi
@@ -9,6 +9,8 @@ require 'whimsy/asf'
ldap = ASF::Person.listids
committers = ASF.committerids # to check for missing ICLAs
+ASF::ICLAFiles.update_cache({})
+
errors = 0
_html do
@@ -27,12 +29,12 @@ _html do
_label "#{ldap.length} People entries found."
_br
_label "#{committers.length} committers found."
- end
+ end
_h2_ 'Error Status'
_div do
_label "#{errors} errors found."
- end
+ end
_h2_ 'Show'
_div do
@@ -71,7 +73,7 @@ _html do
end
iclas = Hash.new{|h,k| h[k]=[]}
- dupes=0
+ dupes = 0
ASF::ICLAFiles.listnames.each do |file|
name = File.basename(file)
stem = name.sub(/\.\w+$/, '')
@@ -79,9 +81,9 @@ _html do
iclas[stem] << name
end
- seen=Hash.new{ |h,k| h[k] = []} # iclas.txt CLA stem values (value = id,name)
- icla_ids=Hash.new{ |h,k| h[k] = []} # to check for duplicates and missing
entries
- icla_mails=Hash.new{ |h,k| h[k] = []} # to check for duplicates
+ seen = Hash.new { |h, k| h[k] = []} # iclas.txt CLA stem values (value =
id,name)
+ icla_ids = Hash.new { |h, k| h[k] = []} # to check for duplicates and
missing entries
+ icla_mails = Hash.new { |h, k| h[k] = []} # to check for duplicates
_h2_ 'Issues'
@@ -229,7 +231,7 @@ _html do
iclas.reject! {|path| seen.include? path}
# select entries with count != 1
- seen.select! {|k,v| v.length != 1}
+ seen.select! { |k, v| v.length != 1}
if seen.size > 0
_h2_ 'Duplicate stem entries in iclas.txt'
_table_ do
@@ -318,7 +320,7 @@ _html do
end
# Check if there are any duplicate mails
- mdups=icla_mails.select{|k,v| v.size > 1}
+ mdups = icla_mails.select {|_k, v| v.size > 1}
if mdups.size > 0
_h2_ 'Duplicate mails in iclas.txt'
_table do
diff --git a/www/secretary/ldap-check.cgi b/www/secretary/ldap-check.cgi
index c451cc54..f2df4dc9 100755
--- a/www/secretary/ldap-check.cgi
+++ b/www/secretary/ldap-check.cgi
@@ -23,6 +23,8 @@ require 'whimsy/asf'
require 'whimsy/asf/mlist'
require 'wunderbar'
+ASF::ICLAFiles.update_cache({})
+
_html do
_style %{
table {border-collapse: collapse}
@@ -178,7 +180,7 @@ _html do
_h2 'People who are banned or have nologin but are still committers'
- people_projects=Hash.new{|h,k| h[k]=Array.new}
+ people_projects = Hash.new {|h, k| h[k] = Array.new}
projects.keys.each do |prj|
prj.members.each do |m|
@@ -191,11 +193,11 @@ _html do
_table do
_tr do
- _th 'UID'
- _th 'Created'
- _th 'asf-banned?'
- _th 'Login'
- _th 'Projects (if any)'
+ _th 'UID'
+ _th 'Created'
+ _th 'asf-banned?'
+ _th 'Login'
+ _th 'Projects (if any)'
end
people.select {|p| p.inactive? and new.include?
p.name}.sort_by(&:name).each do |p|
_tr do
@@ -214,9 +216,9 @@ _html do
_table do
_tr do
- _th 'UID'
- _th 'Created'
- _th 'Login'
+ _th 'UID'
+ _th 'Created'
+ _th 'Login'
end
people.select {|p| p.inactive? and not p.asf_banned?}.sort_by(&:name).each
do |p|
_tr do
@@ -269,17 +271,17 @@ _html do
_p do
_ 'The following ids are in the new group but not the old'
_br
- _ new_old.map{|x| x.inspect}.join(',')
+ _ new_old.map(&:inspect).join(',')
end
elsif old_new.size == 0
- _p 'The groups are equal'
+ _p 'The groups are equal'
end
if old_new.size > 0
_p do
_ 'The following ids are in the old group but not the new'
_br
- _ old_new.map{|x| x.inspect}.join(',')
+ _ old_new.map(&:inspect).join(',')
end
end
diff --git a/www/secretary/ldap-names.cgi b/www/secretary/ldap-names.cgi
index 67a7118b..74d8ddd2 100755
--- a/www/secretary/ldap-names.cgi
+++ b/www/secretary/ldap-names.cgi
@@ -14,6 +14,8 @@ require 'whimsy/asf'
require 'wunderbar/script'
require 'ruby2js/filter/functions'
+ASF::ICLAFiles.update_cache({})
+
_html do
_style %{
table {border-collapse: collapse}
diff --git a/www/secretary/memapp_check.cgi b/www/secretary/memapp_check.cgi
index ece50a9f..02b61926 100755
--- a/www/secretary/memapp_check.cgi
+++ b/www/secretary/memapp_check.cgi
@@ -28,6 +28,9 @@ members << PersonNoId.new("Shane Caraveo")
members << PersonNoId.new("Robert Hartill")
members << PersonNoId.new("Andrew Wilson")
+ASF::MemApps.update_cache({})
+ASF::ICLAFiles.update_cache({})
+
files = Hash[ASF::MemApps.names.map{|i| [i,'NAK']}]
nofiles = Hash.new()
diff --git a/www/secretary/public-names.cgi b/www/secretary/public-names.cgi
index bbe05153..af1df58f 100755
--- a/www/secretary/public-names.cgi
+++ b/www/secretary/public-names.cgi
@@ -14,6 +14,8 @@ unless user.asf_member? or ASF.pmc_chairs.include? user
exit
end
+ASF::ICLAFiles.update_cache({})
+
# default HOME directory
require 'etc'
ENV['HOME'] ||= Etc.getpwuid.dir
diff --git a/www/secretary/workbench/server.rb
b/www/secretary/workbench/server.rb
index 6a4ac452..f5a65e99 100644
--- a/www/secretary/workbench/server.rb
+++ b/www/secretary/workbench/server.rb
@@ -298,6 +298,7 @@ end
# redirect to an icla
get %r{/icla/(.*)} do |filename|
checkout = ASF::SVN.svnurl('iclas')
+ ASF::ICLAFiles.update_cache(env)
file = ASF::ICLAFiles.match_claRef(filename)
pass unless file
redirect to(checkout + '/' + file)
diff --git a/www/secretary/workbench/views/actions/ccla.json.rb
b/www/secretary/workbench/views/actions/ccla.json.rb
index 35062a1c..25599fe4 100644
--- a/www/secretary/workbench/views/actions/ccla.json.rb
+++ b/www/secretary/workbench/views/actions/ccla.json.rb
@@ -15,6 +15,7 @@ ccla = "#@filename#{fileext}"
# verify that a CCLA under that name doesn't already exist
if ccla =~ /\A\w[-\w]*\.?\w*\z/
+ ASF::CCLAFiles.update_cache(env)
if ASF::CCLAFiles.exist?(ccla)
_warn "documents/cclas/#{ccla} already exists"
end
diff --git a/www/secretary/workbench/views/actions/grant.json.rb
b/www/secretary/workbench/views/actions/grant.json.rb
index f1696bdf..bc4173c0 100644
--- a/www/secretary/workbench/views/actions/grant.json.rb
+++ b/www/secretary/workbench/views/actions/grant.json.rb
@@ -15,6 +15,7 @@ grant = "#@filename#{fileext}"
# verify that a grant under that name doesn't already exist
if grant =~ /^\w[-\w]*\.?\w*$/
+ ASF::GrantFiles.update_cache(env)
if ASF::GrantFiles.exist?(grant)
_warn "documents/grants/#{grant} already exists"
end
diff --git a/www/secretary/workbench/views/actions/memapp.json.rb
b/www/secretary/workbench/views/actions/memapp.json.rb
index 2511af61..49aab6d3 100644
--- a/www/secretary/workbench/views/actions/memapp.json.rb
+++ b/www/secretary/workbench/views/actions/memapp.json.rb
@@ -15,6 +15,7 @@ fileext = File.extname(@selected).downcase if
@signature.empty?
# verify that a membership form under that name stem doesn't already exist
if "#{@filename}#{fileext}" =~ /\A\w[-\w]*\.?\w*\z/ # check taint requirements
# returns name if it matches as stem or fully (e.g. for directory)
+ ASF::MemApps.update_cache(env)
form = ASF::MemApps.search @filename
if form
_warn "documents/member_apps/#{form} already exists"
