Update of
/var/cvs/contributions/CMSContainer_Modules/staticdownload/src/java/com/finalist/cmsc/staticdownload
In directory
james.mmbase.org:/tmp/cvs-serv26795/src/java/com/finalist/cmsc/staticdownload
Modified Files:
DownloadSettings.java DownloadThread.java
Log Message:
CMSC-945(Add a field in the admin.jsp which is needed to put webapp
name,perfect the staticdownload module to make it download pictures in css ,js
and html ,also modify the paths of them in downloaded html.)
See also:
http://cvs.mmbase.org/viewcvs/contributions/CMSContainer_Modules/staticdownload/src/java/com/finalist/cmsc/staticdownload
See also: http://www.mmbase.org/jira/browse/CMSC-945
Index: DownloadSettings.java
===================================================================
RCS file:
/var/cvs/contributions/CMSContainer_Modules/staticdownload/src/java/com/finalist/cmsc/staticdownload/DownloadSettings.java,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -b -r1.3 -r1.4
--- DownloadSettings.java 26 Nov 2007 11:42:08 -0000 1.3
+++ DownloadSettings.java 20 Jun 2008 06:53:21 -0000 1.4
@@ -14,9 +14,10 @@
private String wgetPath;
private String downloadUrl;
private ServletContext servletContext;
+ private String webappName;
- public DownloadSettings(int level, String targetPath, String storePath,
String wgetPath, String downloadUrl,
+ public DownloadSettings(int level, String targetPath, String storePath,
String wgetPath, String downloadUrl,String webappName,
ServletContext servletContext) {
this.level = level;
this.tempPath = targetPath;
@@ -24,6 +25,7 @@
this.wgetPath = wgetPath;
this.downloadUrl = downloadUrl;
this.servletContext = servletContext;
+ this.webappName = webappName;
}
@@ -56,4 +58,11 @@
return servletContext;
}
+
+public String getWebappName() {
+ return webappName;
+}
+
+
+
}
Index: DownloadThread.java
===================================================================
RCS file:
/var/cvs/contributions/CMSContainer_Modules/staticdownload/src/java/com/finalist/cmsc/staticdownload/DownloadThread.java,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -b -r1.4 -r1.5
--- DownloadThread.java 12 Apr 2008 14:26:34 -0000 1.4
+++ DownloadThread.java 20 Jun 2008 06:53:21 -0000 1.5
@@ -8,25 +8,34 @@
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
+import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
+import java.io.StringReader;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import net.sf.mmapps.modules.cloudprovider.CloudProviderFactory;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.mmbase.bridge.Cloud;
import org.mmbase.bridge.Node;
+
+
public class DownloadThread extends Thread {
private static Log log = LogFactory.getLog(DownloadThread.class);
@@ -39,6 +48,10 @@
private long endTime = -1;
private String fileName;
private Node node;
+ private static ArrayList<String> redownloadfiles = new ArrayList<String>();
+ private String webPath;
+ private String webappName="";
+ private String[] suffix;
private static final String[] EXCLUDE_FILES = new String[] { "admin",
"data", "editors", "htmlarea", "jsp",
"META-INF", "mmbase", "WEB-INF", "xinha" };
@@ -52,6 +65,7 @@
super("Downloading " + url);
this.url = url;
this.downloadSettings = downloadSettings;
+ this.webappName = downloadSettings.getWebappName();
}
@@ -61,11 +75,35 @@
super.start();
}
-
+ /**Set the suffix according to the txt file*/
+ private void setupSuffix() throws IOException{
+ InputStream is =
Thread.currentThread().getContextClassLoader().getResourceAsStream("../config/subfix.txt");
+ BufferedReader br = new BufferedReader(new InputStreamReader(is));
+ StringBuffer buffer = new StringBuffer();
+ String line;
+ try {
+ while (StringUtils.isNotEmpty(line=br.readLine())){
+ buffer.append(line);
+ }
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ log.info("Some errors happened while reading the subfix.txt");
+ }finally{
+ if(is!=null){
+ is.close();
+ }
+ }
+ suffix = buffer.toString().split(";");
+ }
public void run() {
try {
cleanUp();
download();
+ File file = new File(downloadSettings.getStorePath());//get the files
which wget last download
+ setupSuffix();
+ modifyDownloadPath();
+ findAssociatdFiles(file, new MyFilenameFilter(".css", ".js",
".html"));
+ redownload(redownloadfiles);
zip();
}
catch (Exception e) {
@@ -229,12 +267,24 @@
}
}
-
- private void download() throws IOException {
-
+ /** make the web address useful*/
+ private void modifyDownloadPath(){
+ if(!downloadSettings.getDownloadUrl().endsWith("/")){
+ webPath = downloadSettings.getDownloadUrl()+"/";
+ }else{
+ webPath = downloadSettings.getDownloadUrl();
+ }
+ }
+ /**redownload the things which the before had not done*/
+ private void redownload(List<String> paths) throws IOException{
+ for(String path:paths){
+ pubDownload(path);
+ }
+ }
+ private void pubDownload(String urll) throws IOException{
List<String> command = new ArrayList<String>();
command.add(downloadSettings.getWgetPath());
- command.add(url);
+ command.add(urll);
for (String option : DownloadSettings.WGET_OPTIONS) {
command.add(option);
}
@@ -275,6 +325,10 @@
}
+ private void download() throws IOException {
+ pubDownload(url);
+ }
+
private void processErrors(String text) {
@@ -369,7 +423,216 @@
return string;
}
+ /**
+ * the method is used to filter files by the suffixes
+ */
+ static class MyFilenameFilter implements FilenameFilter {
+ private String suffix = "";
+ private String suffix1 = "";
+ private String suffix2 = "";
+
+ public MyFilenameFilter(String suffix, String suffix1, String suffix2) {
+ this.suffix = suffix;
+ this.suffix1 = suffix1;
+ this.suffix2 = suffix2;
+ }
+
+ public boolean accept(File dir, String name) {
+ // TODO Auto-generated method stub
+ if (new File(dir, name).isFile()) {
+ return name.endsWith(suffix) || name.endsWith(suffix1)
+ || name.endsWith(suffix2);
+ } else {
+ return true;
+ }
+ }
+ }
+ private void spellPath(String targetPath,String directoryName){
+ String regEx1 = "\\.\\.";
+ Pattern p1 = Pattern.compile(regEx1);
+ Matcher m1 = p1.matcher(targetPath);
+ String str1 = m1.replaceAll(directoryName);
+ String path = webPath + str1;
+ if(!redownloadfiles.contains(path))
+ redownloadfiles.add(path);
+ }
+
+ private String getUrl(){
+ return downloadSettings.getDownloadUrl();
+ }
+ /**Use the filter to find the associated files which last wget downloads,
+ * then pick up the getTargetStringAndWriteFile() method
+ * @throws IOException */
+ private void findAssociatdFiles(File file , MyFilenameFilter myfilter)
throws IOException {
+ String fileName = file.getAbsolutePath();
+ if (!fileName.endsWith(File.separator)) {
+ fileName = fileName + File.separator;
+ }
+ if (!file.exists()) {
+ return;
+ }
+ File[] files = file.listFiles(myfilter);
+ for (int i = 0; i < files.length; i++) {
+ if (files[i].isFile()) {
+ for(String suf:suffix){
+ String outputData =
this.getTargetStringAndWriteFile(files[i],suf);
+ if(outputData.length()!=0 &&
files[i].length()!=outputData.length()){
+ try {
+ this.writeFile(files[i],outputData);
+ } catch (IOException e) {
+ log.info("Something wrong in writing data into files");
+ }
+ }
+ }
+ } else if (files[i].isDirectory()) {
+ findAssociatdFiles(files[i], myfilter);
+ }
+ }
+ }
+ /**Regular expressions for judging*/
+ private String regularJudge(String regEx,String targetString){
+ String targetPath = "";
+ if(regEx!=null&&targetString!=null){
+ Pattern p = Pattern.compile(regEx);
+ Matcher m = p.matcher(targetString);
+ boolean rs = m.find();
+ if(rs){
+ targetPath = m.group();
+ }
+ }
+ return targetPath;
+ }
+ /**Regular expressions for replacing*/
+ private String regularReplace(String target,String former,String newer){
+ String regEx = former;
+ Pattern p = Pattern.compile(regEx);
+ Matcher m = p.matcher(target);
+ return m.replaceAll(newer);
+ }
+ /**
+ *Spell the path of downloading , modify the paths in files
+ [EMAIL PROTECTED] targetStringNew : the path of pictures in former file
+ [EMAIL PROTECTED] tempString : the string in file at current line
+ [EMAIL PROTECTED] file : the current file
+ [EMAIL PROTECTED] the new string in current line
+ */
+ private String downloadAndModifyPath(String targetStringNew,String
tempString,File file){
+ String downloadPath;
+ if(StringUtils.isEmpty(webappName)){
+ downloadPath = webPath+targetStringNew.substring(1);
+ }else{
+ downloadPath =
webPath+targetStringNew.substring(webappName.length()+2);
+ }
+ if(!redownloadfiles.contains(downloadPath))
+ redownloadfiles.add(downloadPath);
+ String filePath = file.getParent();
+ int beginIndex = filePath.indexOf(getUrl());
+ String filePathIntercept =
filePath.substring(beginIndex+getUrl().length());
+ String[] strs = filePathIntercept.split("\\\\");
+ int num = strs.length;
+ String docs ="";
+ for(int i=1;i< num;i++){
+ docs += "../";
+ }
+
if(StringUtils.isEmpty(filePathIntercept)&&StringUtils.isEmpty(webappName)){
+ String targetStringNews =targetStringNew.substring(1);
+ tempString = this.regularReplace(tempString, targetStringNew,
targetStringNews);
+ }else
if(StringUtils.isNotEmpty(filePathIntercept)&&StringUtils.isEmpty(webappName)){
+ tempString = this.regularReplace(tempString, targetStringNew,
docs+targetStringNew.substring(1));
+ }else
if(StringUtils.isEmpty(filePathIntercept)&&StringUtils.isNotEmpty(webappName)){
+ String targetStringNews
=targetStringNew.substring(webappName.length()+2);
+ tempString = this.regularReplace(tempString, targetStringNew,
targetStringNews);
+ }else
if(StringUtils.isNotEmpty(filePathIntercept)&&StringUtils.isNotEmpty(webappName)){
+ tempString = this.regularReplace(tempString, "/"+webappName+"/",
docs);
+ }
+ return tempString;
+ }
+ private void addDownloadPath(String str,File file,String targetPath){
+ String filepath = file.getAbsolutePath();
+ String url = downloadSettings.getDownloadUrl();
+ int tagStart=filepath.indexOf(url);
+ String u = filepath.substring(tagStart);
+ int tagEnd = u.indexOf("\\"+str);
+ String m = u.substring(0,tagEnd);
+ String ss = regularReplace(m, "\\\\", "/");
+ String str1 = regularReplace(targetPath, "\\.\\.", ss);
+ if(!redownloadfiles.contains(str1))
+ redownloadfiles.add(str1);
+ }
+ /**
+ * Use regular expressions to get the paths of pictures
+ * and rewrite the path into the file
+ * @param file : the file to be read
+ * @param suffix : the messages about suffix of pictures
+ * @throws IOException
+ */
+ private String getTargetStringAndWriteFile(File file,String suffix) throws
IOException{
+ BufferedReader reader = null;
+ String outputData = "";
+ try{
+ reader = new BufferedReader(new FileReader(file));
+ String tempString = null;
+ while((tempString =
reader.readLine())!=null&&!file.getName().endsWith(".html")){
+ String regEx = "[\\w/.-]+\\."+suffix;
+ String targetPath = regularJudge(regEx,tempString);
+ if(targetPath!=""){
+ int tagStart = -1;
+ String fileParentPath = file.getParent()+File.separator;
+ String directoryName ="";
+ if(StringUtils.isNotEmpty(webappName)&&(tagStart =
fileParentPath.indexOf(webappName))>=0&&fileParentPath.contains("js")){
+ int tagEnd =
fileParentPath.indexOf("\\js",tagStart+webappName.length()+1);
+ if(tagEnd>=0){
+ directoryName =
fileParentPath.substring(tagStart+webappName.length()+1, tagEnd);
+ this.spellPath(targetPath,directoryName);
+ }
+ }else if(StringUtils.isNotEmpty(webappName)&&(tagStart
=fileParentPath.indexOf(webappName))>=0&&fileParentPath.contains("css")){
+ int tagEnd =
fileParentPath.indexOf("\\css",tagStart+webappName.length()+1);
+ if(tagEnd>=0){
+ directoryName =
fileParentPath.substring(tagStart+webappName.length()+1, tagEnd);
+ this.spellPath(targetPath,directoryName);
+ }
+ }else
if(StringUtils.isEmpty(webappName)&&fileParentPath.contains("css")){
+ addDownloadPath("css",file,targetPath);
+ }else
if(StringUtils.isEmpty(webappName)&&fileParentPath.contains("js")){
+ addDownloadPath("js",file,targetPath);
+ }
+ }
+ }
+ while((tempString =
reader.readLine())!=null&&file.getName().endsWith(".html")){
+ String targetString =
regularJudge("(?m)^\\s*var.*"+suffix+".*$", tempString);
+ if(StringUtils.isNotEmpty(targetString)){
+ String targetStringNew =
regularJudge("[\\w/\\.-]+\\."+suffix,targetString);
+ if(StringUtils.isNotEmpty(targetStringNew)){
+
if(targetStringNew.startsWith("/"+webappName)&&StringUtils.isNotEmpty(webappName)){
+ tempString =
this.downloadAndModifyPath(targetStringNew, tempString, file);
+ }else
if(targetStringNew.startsWith("/")&&StringUtils.isEmpty(webappName)){
+ tempString =
this.downloadAndModifyPath(targetStringNew, tempString, file);
+ }
+ }
+ }
+ String str =
regularJudge("[\\s.]*background-image:\\s*url.*", tempString);
+ if(StringUtils.isNotEmpty(str)){
+ String targetName = regularJudge("[\\w/]*\\."+suffix,
str);
+ if(StringUtils.isNotEmpty(targetName)){
+
if(targetName.startsWith("/"+webappName)&&StringUtils.isNotEmpty(webappName)){
+ tempString = this.downloadAndModifyPath(targetName,
tempString, file);
+ }else
if(targetName.startsWith("/")&&StringUtils.isEmpty(webappName)){
+ tempString = this.downloadAndModifyPath(targetName,
tempString, file);
+ }
+ }
+ }
+ outputData+=tempString+"\r\n";
+ }
+ }catch (IOException e) {
+ log.error("Something wrong while reading ,",e);
+ }finally{
+ if(reader!=null){
+ reader.close();
+ }
+ }
+ return outputData;
+ }
public boolean isDownloading() {
return (endTime == -1);
}
_______________________________________________
Cvs mailing list
[email protected]
http://lists.mmbase.org/mailman/listinfo/cvs
