Upayavira wrote: > James Bates wrote: > > The Cocoon command line interface provides a switch for simulating the > > Cocoon User-Agent header that would be sent by a browser. The idea being > > that it could be used by e.g. the browser selector to "detect" that a > > request is coming from the CLI. > > > > > > > > When investigating however, I noticed that the Cocoon bean (the class > > that implements the CLI) does not place the User-Agent into a HEDAER, > > but into a request PARAMETER instead (occurs on line 407 of > > CocoonWrapper.java, in method processURI() in BRANCH_2_1_X; line 421 of > > the same file in 2.2 trunk). > > > > > > > > Is there a particular reason for this behaviour? Would it be desirable > > to change the behaviour so the User-Agent is indeed submitted as a > > request HEADER, thus perfectly emulating a HTTP call? I am perfectly
> > willing to submit a patch to that effect... > > The CLI/bean should work exactly the same as a browser. I have seen that > on occasions and wondered about it. > > What you are saying makes sense, and I'd happily commit a patch of yours. > > Regards, Upayavira Sorry people; had some trouble subscribing to the mailing list; but it's done now. Below is a "svn patch" against the BRANCH_2_1_X development tree that changes the behaviour as described above... Careful before committing: the code ALTERS the behaviour sothat any existing implemenetations that depend on the user agent being in a PARAMETER and not a HEADER when running the CLI will break... It would be recommended I imagine to mention this in the "Changes" file? Regards, James ---- cli-user-agent-header.patch -------------------------- Index: src/java/org/apache/cocoon/environment/commandline/FileSavingEnvironment .java =================================================================== --- src/java/org/apache/cocoon/environment/commandline/FileSavingEnvironment .java (revision 170000) +++ src/java/org/apache/cocoon/environment/commandline/FileSavingEnvironment .java (working copy) @@ -31,7 +31,7 @@ * * @author <a href="mailto:[EMAIL PROTECTED]">Stefano Mazzocchi</a> * @author <a href="mailto:[EMAIL PROTECTED]">Upayavira</a> - * @version CVS $Id: FileSavingEnvironment.java,v 1.6 2004/03/05 13:02:54 bdelacretaz Exp $ + * @version CVS $Id$ */ public class FileSavingEnvironment extends AbstractCommandLineEnvironment { @@ -43,6 +43,7 @@ File context, Map attributes, Map parameters, + Map headers, Map links, List gatheredLinks, CommandLineContext cliContext, @@ -51,7 +52,7 @@ throws MalformedURLException { super(uri, null, context, stream, log); this.objectModel.put(ObjectModelHelper.REQUEST_OBJECT, - new CommandLineRequest(this, null, uri, null, attributes, parameters)); + new CommandLineRequest(this, null, uri, null, attributes, parameters, headers)); this.objectModel.put(ObjectModelHelper.RESPONSE_OBJECT, new CommandLineResponse()); this.objectModel.put(ObjectModelHelper.CONTEXT_OBJECT, @@ -64,18 +65,19 @@ this.objectModel.put(Constants.LINK_COLLECTION_OBJECT, gatheredLinks); } } - + public FileSavingEnvironment(String uri, File context, Map attributes, Map parameters, + Map headers, Map links, List gatheredLinks, CommandLineContext cliContext, OutputStream stream, Logger log) throws MalformedURLException { - this(uri, 0L, context, attributes, parameters, links, gatheredLinks, cliContext, stream, log); + this(uri, 0L, context, attributes, parameters, headers, links, gatheredLinks, cliContext, stream, log); } /** Index: src/java/org/apache/cocoon/bean/CocoonWrapper.java =================================================================== --- src/java/org/apache/cocoon/bean/CocoonWrapper.java (revision 170000) +++ src/java/org/apache/cocoon/bean/CocoonWrapper.java (working copy) @@ -83,8 +83,8 @@ private String logKit = null; protected String logger = null; protected String logLevel = "ERROR"; - private String userAgent = DEFAULT_USER_AGENT; - private String accept = DEFAULT_ACCEPT; + protected String userAgent = DEFAULT_USER_AGENT; + protected String accept = DEFAULT_ACCEPT; private List classList = new ArrayList(); // Objects used alongside User Supplied Parameters @@ -190,7 +190,7 @@ } }.instance(); } - + protected ExcaliburComponentManager getComponentManager() { return cocoon.getComponentManager(); } @@ -385,7 +385,7 @@ public void setUseExistingCocoon(boolean useExistingCocoon) { this.useExistingCocoon = useExistingCocoon; } - + /** * Process single URI into given output stream. * @@ -402,13 +402,14 @@ // Get parameters, deparameterized URI and path from URI final TreeMap parameters = new TreeMap(); + final TreeMap headers = new TreeMap(); final String deparameterizedURI = NetUtils.deparameterize(uri, parameters); - parameters.put("user-agent", userAgent); - parameters.put("accept", accept); + headers.put("user-agent", userAgent); + headers.put("accept", accept); int status = - getPage(deparameterizedURI, 0L, parameters, null, null, outputStream); + getPage(deparameterizedURI, 0L, parameters, headers, null, null, outputStream); if (status >= 400) { throw new ProcessingException("Resource not found: " + status); @@ -432,13 +433,14 @@ // Get parameters, deparameterized URI and path from URI final TreeMap parameters = new TreeMap(); + final TreeMap headers = new TreeMap(); final String deparameterizedURI = NetUtils.deparameterize(uri, parameters); - parameters.put("user-agent", userAgent); - parameters.put("accept", accept); + headers.put("user-agent", userAgent); + headers.put("accept", accept); int status = - getPage(deparameterizedURI, 0L, parameters, null, null, handler); + getPage(deparameterizedURI, 0L, parameters, headers, null, null, handler); if (status >= 400) { throw new ProcessingException("Resource not found: " + status); @@ -491,17 +493,18 @@ protected int getPage(String deparameterizedURI, long lastModified, Map parameters, + Map headers, Map links, List gatheredLinks, OutputStream stream) throws Exception { - parameters.put("user-agent", userAgent); - parameters.put("accept", accept); + headers.put("user-agent", userAgent); + headers.put("accept", accept); FileSavingEnvironment env = new FileSavingEnvironment(deparameterizedURI, lastModified, context, - attributes, parameters, links, + attributes, parameters, headers, links, gatheredLinks, cliContext, stream, log); // Here Cocoon can throw an exception if there are errors in processing the page @@ -528,17 +531,18 @@ protected int getPage(String deparameterizedURI, long lastModified, Map parameters, + Map headers, Map links, List gatheredLinks, ContentHandler handler) throws Exception { - parameters.put("user-agent", userAgent); - parameters.put("accept", accept); + headers.put("user-agent", userAgent); + headers.put("accept", accept); FileSavingEnvironment env = new FileSavingEnvironment(deparameterizedURI, lastModified, context, - attributes, parameters, links, + attributes, parameters, headers, links, gatheredLinks, cliContext, null, log); XMLConsumer consumer = new ContentHandlerWrapper(handler); @@ -580,12 +584,13 @@ protected String getType(String deparameterizedURI, Map parameters) throws Exception { - parameters.put("user-agent", userAgent); - parameters.put("accept", accept); + TreeMap headers = new TreeMap(); + headers.put("user-agent", userAgent); + headers.put("accept", accept); FileSavingEnvironment env = new FileSavingEnvironment(deparameterizedURI, context, attributes, - parameters, empty, null, cliContext, + parameters, headers, empty, null, cliContext, new NullOutputStream(), log); processLenient(env); return env.getContentType(); Index: src/java/org/apache/cocoon/bean/CocoonBean.java =================================================================== --- src/java/org/apache/cocoon/bean/CocoonBean.java (revision 170000) +++ src/java/org/apache/cocoon/bean/CocoonBean.java (working copy) @@ -1,12 +1,12 @@ /* * Copyright 1999-2004 The Apache Software Foundation. - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -47,6 +47,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.TreeMap; /** * <p>The Cocoon Bean simplifies usage of the Cocoon object. Allows to create, @@ -81,7 +82,7 @@ private boolean verbose; SourceResolver sourceResolver; - private Crawler crawler; + private Crawler crawler; private String checksumsURI = null; private Map checksums; @@ -129,7 +130,7 @@ public boolean isPrecompileOnly() { return precompileOnly; } - + public void setVerbose(boolean verbose) { this.verbose = verbose; } @@ -149,7 +150,7 @@ public void setChecksumURI(String uri) { this.checksumsURI = uri; } - + public boolean followLinks() { return followLinks; } @@ -241,7 +242,7 @@ public int getTargetCount() { return crawler.getRemainingCount(); } - + public void addExcludePattern(String pattern) { int preparedPattern[] = WildcardHelper.compilePattern(pattern); excludePatterns.add(preparedPattern); @@ -345,7 +346,7 @@ if (this.checksumsURI != null) { readChecksumFile(); } - + if (crawler.getRemainingCount()>=0) { Iterator iterator = crawler.iterator(); while (iterator.hasNext()) { @@ -355,11 +356,11 @@ } } } - + if (this.checksumsURI != null) { writeChecksumFile(); } - + if (log.isInfoEnabled()) { log.info( " Memory used: " @@ -490,11 +491,17 @@ gatheredLinks = null; } + + TreeMap headers = new TreeMap(); + headers.put("user-agent", userAgent); + headers.put("accept", accept); + status = getPage( target.getDeparameterizedSourceURI(), getLastModified(target), target.getParameters(), + headers, target.confirmExtensions() ? translatedLinks : null, gatheredLinks, output); @@ -537,14 +544,14 @@ ModifiableSource source = getSource(target); try { pageSize = output.size(); - + if (this.checksumsURI == null || !isSameContent(output, target)) { OutputStream stream = source.getOutputStream(); output.setFileOutputStream(stream); output.flush(); output.close(); - pageGenerated(target.getSourceURI(), - target.getAuthlessDestURI(), + pageGenerated(target.getSourceURI(), + target.getAuthlessDestURI(), pageSize, linkCount, newLinkCount, @@ -668,11 +675,11 @@ } /* NB. This is a temporary solution - it may well be replaced by storing the checksum info - * in the XML 'report' file, along with details of what pages were created, etc. - */ + * in the XML 'report' file, along with details of what pages were created, etc. + */ private void readChecksumFile() throws Exception { checksums = new HashMap(); - + try { Source checksumSource = sourceResolver.resolveURI(checksumsURI); BufferedReader reader = new BufferedReader(new InputStreamReader(checksumSource.getInputStream())); @@ -683,7 +690,7 @@ if (line.trim().startsWith("#") || line.trim().length()==0 ) { continue; } - if (line.indexOf("\t")==-1) { + if (line.indexOf("\t")==-1) { throw new ProcessingException("Missing tab at line "+lineNo+" of " + checksumsURI); } String filename = line.substring(0,line.indexOf("\t")); @@ -695,7 +702,7 @@ // return leaving checksums map empty } } - + private void writeChecksumFile() throws Exception { Source checksumSource = sourceResolver.resolveURI(checksumsURI); if (!(checksumSource instanceof ModifiableSource)) { @@ -718,7 +725,7 @@ md5.update(stream.getContent()); String streamDigest = SourceUtil.encodeBASE64(new String(md5.digest())); String targetDigest = (String)checksums.get(target.getSourceURI()); - + if (streamDigest.equals(targetDigest)) { return true; } else {
