[whimsy] 01/02: Extend scanner to show raw URLs

[sebb]

This is an automated email from the ASF dual-hosted git repository.

sebb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/whimsy.git

commit 34f37cbf628bb372913a0fed8327a957aee8ee70
Author: Sebb <s...@apache.org>
AuthorDate: Tue May 3 23:15:00 2022 +0100

    Extend scanner to show raw URLs
---
 tools/scan-page.js | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tools/scan-page.js b/tools/scan-page.js
index 811b082c..55b06cdc 100755
--- a/tools/scan-page.js
+++ b/tools/scan-page.js
@@ -7,9 +7,10 @@ module.paths.push('/usr/lib/node_modules')
 const puppeteer = require('puppeteer');
 
 const target = process.argv[2] || 'http://apache.org/';
+const option = process.argv[3] || '';
 
 function isASFhost(host) {
-    return host == 'apache.org' || host.endsWith('.apache.org') || 
host.endsWith('.apachecon.com');
+    return host == '' || host == 'apache.org' || host.endsWith('.apache.org') 
|| host.endsWith('.apachecon.com');
 }
 
 (async () => {
@@ -26,10 +27,15 @@ function isASFhost(host) {
         interceptedRequest.continue();
     } else {
         let host = new URL(url).host
-        // don't visit non-ASF hosts
         if (!isASFhost(host)) {
-            console.log(host);
-            interceptedRequest.abort();
+            // don't visit non-ASF hosts unless requested
+            if (option == 'all') {
+                console.log(url);
+                interceptedRequest.continue();
+            } else {
+                console.log(host);
+                interceptedRequest.abort();
+            }
         } else { 
             // Need to visit at least an initial redirect
             interceptedRequest.continue();