Modified: tapestry/tapestry-site/trunk/src/main/java/org/apache/cxf/cwiki/SiteExporter.java URL: http://svn.apache.org/viewvc/tapestry/tapestry-site/trunk/src/main/java/org/apache/cxf/cwiki/SiteExporter.java?rev=1525452&r1=1525451&r2=1525452&view=diff ============================================================================== --- tapestry/tapestry-site/trunk/src/main/java/org/apache/cxf/cwiki/SiteExporter.java (original) +++ tapestry/tapestry-site/trunk/src/main/java/org/apache/cxf/cwiki/SiteExporter.java Sun Sep 22 21:33:09 2013 @@ -19,11 +19,11 @@ package org.apache.cxf.cwiki; - import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileWriter; +import java.io.IOException; import java.io.InputStream; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; @@ -47,12 +47,16 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CopyOnWriteArraySet; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicInteger; import javax.xml.datatype.DatatypeFactory; import javax.xml.datatype.XMLGregorianCalendar; import javax.xml.namespace.QName; +import javax.xml.parsers.ParserConfigurationException; import javax.xml.ws.AsyncHandler; import javax.xml.ws.Dispatch; import javax.xml.ws.Response; @@ -81,7 +85,7 @@ import org.apache.cxf.transport.http.HTT import org.apache.cxf.transports.http.configuration.HTTPClientPolicy; import org.apache.velocity.Template; import org.apache.velocity.VelocityContext; -import org.apache.velocity.app.Velocity; +import org.apache.velocity.app.VelocityEngine; import org.apache.velocity.runtime.resource.loader.URLResourceLoader; import org.ccil.cowan.tagsoup.Parser; import org.ccil.cowan.tagsoup.XMLWriter; @@ -93,18 +97,19 @@ public class SiteExporter implements Run static final String HOST = "https://cwiki.apache.org";; static final String ROOT = HOST + "/confluence"; - static final String RPC_ROOT = "/rpc/soap-axis/confluenceservice-v1"; + static final String RPC_ROOT = "/rpc/soap-axis/confluenceservice-v"; static final String SOAPNS = "http://soap.rpc.confluence.atlassian.com";; + static final String SEPARATOR = " > "; - static final String CONFLUENCE_IMAGES = "images/confluence"; - static boolean debug; static String userName = "cxf-export-user"; static String password; + static int apiVersion = 1; + static boolean svn; static boolean commit; static StringBuilder svnCommitMessage = new StringBuilder(); @@ -113,11 +118,16 @@ public class SiteExporter implements Run static String loginToken; static Dispatch<Document> dispatch; static AtomicInteger asyncCount = new AtomicInteger(); + static Map<String, Space> spaces = new ConcurrentHashMap<String, Space>(); + static List<SiteExporter> siteExporters; Map<String, Page> pages = new ConcurrentHashMap<String, Page>(); Collection<Page> modifiedPages = new ConcurrentLinkedQueue<Page>(); Set<String> globalPages = new CopyOnWriteArraySet<String>(); - Set<String> blogPages = new CopyOnWriteArraySet<String>(); + + Map<String, BlogEntrySummary> blog = new ConcurrentHashMap<String, BlogEntrySummary>(); + Set<BlogEntrySummary> modifiedBlog = new CopyOnWriteArraySet<BlogEntrySummary>(); + String spaceKey = "CXF"; String pageCacheFile = "pagesConfig.obj"; @@ -129,7 +139,9 @@ public class SiteExporter implements Run File outputDir = rootOutputDir; Template template; + Space space; + public SiteExporter(String fileName, boolean force) throws Exception { forceAll = force; @@ -159,29 +171,25 @@ public class SiteExporter implements Run String[] pgs = globals.split(","); globalPages.addAll(Arrays.asList(pgs)); } - if (props.containsKey("blogPages")) { - String blogpages = props.getProperty("blogPages"); - String[] pgs = blogpages.split(","); - blogPages.addAll(Arrays.asList(pgs)); - } props = new Properties(); String clzName = URLResourceLoader.class.getName(); props.put("resource.loader", "url"); props.put("url.resource.loader.class", clzName); props.put("url.resource.loader.root", ""); - synchronized (Velocity.class) { - Velocity.init(props); + + VelocityEngine engine = new VelocityEngine(); + engine.init(props); - URL url = ClassLoaderUtils.getResource(templateName, this.getClass()); - if (url == null) { - File file = new File(templateName); - if (file.exists()) { - url = file.toURI().toURL(); - } + URL url = ClassLoaderUtils.getResource(templateName, this.getClass()); + if (url == null) { + File file = new File(templateName); + if (file.exists()) { + url = file.toURI().toURL(); } - template = Velocity.getTemplate(url.toURI().toString()); } + template = engine.getTemplate(url.toURI().toString()); + outputDir.mkdirs(); } @@ -190,7 +198,7 @@ public class SiteExporter implements Run Service service = Service.create(new QName(SOAPNS, "Service")); service.addPort(new QName(SOAPNS, "Port"), SOAPBinding.SOAP11HTTP_BINDING, - ROOT + RPC_ROOT); + ROOT + RPC_ROOT + apiVersion); dispatch = service.createDispatch(new QName(SOAPNS, "Port"), Document.class, Service.Mode.PAYLOAD); @@ -216,7 +224,7 @@ public class SiteExporter implements Run public void run() { try { - doExport(); + render(); } catch (Exception e) { e.printStackTrace(); } @@ -226,29 +234,70 @@ public class SiteExporter implements Run Page p = findPage(s); if (p != null) { pages.remove(p.getId()); - modifiedPages.add(p); + if (!modifiedPages.contains(p)) { + modifiedPages.add(p); + } } } - public void doExport() throws Exception { + /** + * @return true if some pages have changed - rendering is needed + * @throws Exception + */ + public boolean initialize() throws Exception { if (!forceAll) { - loadPagesCache(); + loadCache(); } - + // debug stuff, force regen of a page //forcePage("Navigation"); //forcePage("Index"); //forcePage("JavaDoc"); //forcePage("DOSGi Architecture"); + //forcePage("Book In One Page"); if (modifiedPages.isEmpty() && checkRSS()) { System.out.println("(" + spaceKey + ") No changes detected from RSS"); - return; + return false; } - + doLogin(); + checkVersion(); + getSpace(); + if ("-space-".equals(breadCrumbRoot)) { + breadCrumbRoot = space.getName(); + } + loadBlog(); loadPages(); + return true; + } + + private void checkVersion() throws ParserConfigurationException, IOException { + Document doc = DOMUtils.createDocument(); + Element el = doc.createElementNS(SOAPNS, "ns1:getServerInfo"); + Element el2 = doc.createElement("in0"); + el.appendChild(el2); + el2.setTextContent(loginToken); + doc.appendChild(el); + + doc = getDispatch().invoke(doc); + el = DOMUtils.getFirstElement(DOMUtils.getFirstElement(doc.getDocumentElement())); + while (el != null) { + if ("majorVersion".equals(el.getLocalName())) { + String major = DOMUtils.getContent(el); + if (Integer.parseInt(major) >= 5) { + apiVersion = 2; + ((java.io.Closeable)dispatch).close(); + dispatch = null; + } + } + + el = DOMUtils.getNextElement(el); + } + } + + protected void render() throws Exception { for (Page p : modifiedPages) { if (globalPages.contains(p.getTitle())) { modifiedPages.clear(); @@ -256,18 +305,27 @@ public class SiteExporter implements Run break; } } + if (forceAll) { modifiedPages.clear(); modifiedPages.addAll(pages.values()); + + modifiedBlog.clear(); + modifiedBlog.addAll(blog.values()); } - - - if (!modifiedPages.isEmpty()) { + if (!modifiedBlog.isEmpty()) { + //blogs changed, see if any pages have blogs + for (Page p : pages.values()) { + if (p.hasBlog() && !modifiedPages.contains(p)) { + modifiedPages.add(p); + } + } + } + if (!modifiedPages.isEmpty() || !modifiedBlog.isEmpty()) { + renderBlog(); renderPages(); - savePages(); + saveCache(); } - - } @@ -286,15 +344,15 @@ public class SiteExporter implements Run List<Element> els = DOMUtils.getChildrenWithName(doc.getDocumentElement(), "http://www.w3.org/2005/Atom";, "entry"); - //XMLUtils.printDOM(doc); + // XMLUtils.printDOM(doc); for (Element el : els) { Element e2 = DOMUtils.getFirstChildWithName(el, "http://www.w3.org/2005/Atom";, "updated"); String val = DOMUtils.getContent(e2); XMLGregorianCalendar cal = DatatypeFactory.newInstance().newXMLGregorianCalendar(val); e2 = DOMUtils.getFirstChildWithName(el, "http://www.w3.org/2005/Atom";, "title"); String title = DOMUtils.getContent(e2); - Page p = findPage(title); + Page p = findPage(title); if (p != null) { //found a modified page - need to rebuild if (cal.compare(p.getModifiedTime()) > 0) { @@ -302,24 +360,36 @@ public class SiteExporter implements Run return false; } } else { - System.out.println("(" + spaceKey + ") Did not find page for: " + title); - return false; + BlogEntrySummary entry = findBlogEntry(title); + if (entry != null) { + // we don't have modified date so just assume it's modified + // we'll use version number to actually figure out if page is modified or not + System.out.println("(" + spaceKey + ") Possible changed blog page found: " + title); + return false; + } else { + System.out.println("(" + spaceKey + ") Did not find page for: " + title); + return false; + } } } return true; } - private void savePages() throws Exception { + private void saveCache() throws Exception { File file = new File(rootOutputDir, pageCacheFile); file.getParentFile().mkdirs(); FileOutputStream fout = new FileOutputStream(file); ObjectOutputStream oout = new ObjectOutputStream(fout); oout.writeObject(pages); + oout.writeObject(blog); oout.close(); } private void renderPages() throws Exception { + PageManager pageManager = new PageManager(this); + Renderer renderer = new Renderer(this); + int total = modifiedPages.size(); int count = 0; for (Page p : modifiedPages) { @@ -330,9 +400,13 @@ public class SiteExporter implements Run loadPageContent(p, null, null); VelocityContext ctx = new VelocityContext(); - ctx.put("exporter", this); + ctx.put("autoexport", this); ctx.put("page", p); + ctx.put("body", p.getContent()); ctx.put("confluenceUri", ROOT); + ctx.put("pageManager", pageManager); + ctx.put("renderer", renderer); + ctx.put("exporter", this); File file = new File(outputDir, p.createFileName()); boolean isNew = !file.exists(); @@ -348,8 +422,56 @@ public class SiteExporter implements Run } else { svnCommitMessage.append("Modified: " + file.getName() + "\n"); } + + p.setContent(null); + } + } + + private void renderBlog() throws Exception { + PageManager pageManager = new PageManager(this); + Renderer renderer = new Renderer(this); + + int total = modifiedBlog.size(); + int count = 0; + for (BlogEntrySummary entry : modifiedBlog) { + count++; + System.out.println("(" + spaceKey + ") Rendering Blog Entry " + entry.getTitle() + + " (" + count + "/" + total + ")"); + + loadAttachments(entry); + String body = renderPage(entry); + body = updateContentLinks(entry, body, null, mainDivClass); + + pageManager.setDirectory(entry.getDirectory()); + + VelocityContext ctx = new VelocityContext(); + ctx.put("autoexport", this); + ctx.put("page", entry); + ctx.put("body", body); + ctx.put("confluenceUri", ROOT); + ctx.put("pageManager", pageManager); + ctx.put("renderer", renderer); + ctx.put("exporter", this); + ctx.put("isBlogEntry", Boolean.TRUE); + + File file = new File(outputDir, entry.getPath()); + file.getParentFile().mkdirs(); + boolean isNew = !file.exists(); + + FileWriter writer = new FileWriter(file); + ctx.put("out", writer); + template.merge(ctx, writer); + writer.close(); + if (isNew) { + //call "svn add" + callSvn("add", file.getAbsolutePath()); + svnCommitMessage.append("Adding: " + file.getName() + "\n"); + } else { + svnCommitMessage.append("Modified: " + file.getName() + "\n"); + } } } + void callSvn(String ... commands) throws Exception { callSvn(outputDir, commands); } @@ -367,8 +489,8 @@ public class SiteExporter implements Run } } - private void loadAttachments(Page p) throws Exception { - Document doc = XMLUtils.newDocument(); + private void loadAttachments(AbstractPage p) throws Exception { + Document doc = DOMUtils.createDocument(); Element el = doc.createElementNS(SOAPNS, "ns1:getAttachments"); Element el2 = doc.createElement("in0"); el.appendChild(el2); @@ -389,7 +511,7 @@ public class SiteExporter implements Run p.addAttachment(aid, filename); - String dirName = p.createFileName(); + String dirName = p.getPath(); dirName = dirName.substring(0, dirName.lastIndexOf(".")) + ".data"; File file = new File(outputDir, dirName); if (!file.exists()) { @@ -417,37 +539,20 @@ public class SiteExporter implements Run el = DOMUtils.getNextElement(el); } } - String loadIcon(String href) throws Exception { - - String filename = href.replace("/confluence/images", "images/confluence"); - File file = new File(outputDir + filename); - if(file.exists()) - return file.getName(); - if(!file.getParentFile().exists()) { - file.getParentFile().mkdirs(); - } - FileOutputStream out = new FileOutputStream(file); - URL url = new URL(HOST + href); - InputStream ins = url.openStream(); - IOUtils.copy(ins, out); - out.close(); - ins.close(); - return file.getName(); - } - String loadUserImage(Page p, String href) throws Exception { + String loadUserImage(AbstractPage p, String href) throws Exception { return loadPageBinaryData(p, href, "userimage", true); } - String loadThumbnail(Page p, String href) throws Exception { + String loadThumbnail(AbstractPage p, String href) throws Exception { return loadPageBinaryData(p, href, "thumbs", false); } - String loadPageBinaryData(Page p, String href, String type, boolean auth) throws Exception { + String loadPageBinaryData(AbstractPage p, String href, String type, boolean auth) throws Exception { String filename = href.substring(href.lastIndexOf('/') + 1); filename = filename.replace(' ', '_'); if (filename.indexOf('?') != -1) { filename = filename.substring(0, filename.indexOf('?')); } - String dirName = p.createFileName(); + String dirName = p.getPath(); dirName = dirName.substring(0, dirName.lastIndexOf(".")) + "." + type; File file = new File(outputDir, dirName); if (!file.exists()) { @@ -490,21 +595,13 @@ public class SiteExporter implements Run } } public Page findPage(String title) throws Exception { - for (Page p : pages.values()) { - if (title.equals(p.getTitle())) { - return p; - } - } - return null; + return (Page) findByTitle(title, pages.values()); } + public Page findPageByURL(String url) throws Exception { - for (Page p : pages.values()) { - if (p.getURL().endsWith(url)) { - return p; - } - } - return null; + return (Page) findByURL(url, pages.values()); } + public Page findPageByID(String id) { for (Page p : pages.values()) { if (p.getId().equals(id)) { @@ -513,14 +610,46 @@ public class SiteExporter implements Run } return null; } - public String breadcrumbs(Page page) { - String separator = ">"; - String s = " " + separator + " "; + public String breadcrumbs(BlogEntrySummary page) { + StringBuffer buffer = new StringBuffer(); + if (breadCrumbRoot != null) { + buffer.append("<a href=\""); + buffer.append("../../../index.html"); + buffer.append("\">"); + buffer.append(breadCrumbRoot); + buffer.append("</a>"); + buffer.append(SEPARATOR); + } else { + buffer.append("<a href=\"../../../index.html\">Index</a>"); + buffer.append(SEPARATOR); + } + XMLGregorianCalendar published = page.getPublished(); + buffer.append(String.valueOf(published.getYear())); + buffer.append(SEPARATOR); + if (published.getMonth() < 10) { + buffer.append("0"); + } + buffer.append(String.valueOf(published.getMonth())); + buffer.append(SEPARATOR); + if (published.getDay() < 10) { + buffer.append("0"); + } + buffer.append(String.valueOf(published.getDay())); + buffer.append(SEPARATOR); + buffer.append("<a href=\""); + buffer.append(page.createFileName()); + buffer.append("\">"); + buffer.append(page.getTitle()); + buffer.append("</a>"); + return buffer.toString(); + } + + public String breadcrumbs(Page page) { StringBuffer buffer = new StringBuffer(); List<Page> p = new LinkedList<Page>(); String parentId = page.getParentId(); - Page parent = parentId == null ? null : pages.get(parentId); + Page parent = pages.get(parentId); while (parent != null) { p.add(0, parent); parentId = parent.getParentId(); @@ -532,7 +661,7 @@ public class SiteExporter implements Run buffer.append("\">"); buffer.append(breadCrumbRoot); buffer.append("</a>"); - buffer.append(s); + buffer.append(SEPARATOR); } for (Page p2 : p) { buffer.append("<a href=\""); @@ -540,7 +669,7 @@ public class SiteExporter implements Run buffer.append("\">"); buffer.append(p2.getTitle()); buffer.append("</a>"); - buffer.append(s); + buffer.append(SEPARATOR); } buffer.append("<a href=\""); buffer.append(page.createFileName()); @@ -575,7 +704,19 @@ public class SiteExporter implements Run } return p.getContent(); } - private String loadPageContent(Page p, String divId, String divCls) throws Exception { + protected String loadPageContent(Page p, String divId, String divCls) throws Exception { + String content = renderPage(p); + content = updateContentLinks(p, content, divId, + divCls == null && divId == null ? mainDivClass : divCls); + if (divId == null) { + p.setContent(content); + } else { + p.setContentForDivId(divId, content); + } + return content; + } + + private String renderPage(AbstractPage p) throws ParserConfigurationException { Document doc = XMLUtils.newDocument(); Element el = doc.createElementNS(SOAPNS, "ns1:renderContent"); Element el2 = doc.createElement("in0"); @@ -609,24 +750,19 @@ public class SiteExporter implements Run doc.appendChild(el); doc = getDispatch().invoke(doc); - String content = doc.getDocumentElement().getFirstChild().getTextContent().trim(); - content = updateContentLinks(p, content, divId, - divCls == null && divId == null ? mainDivClass : divCls); - if (divId == null) { - p.setContent(content); - } else { - p.setContentForDivId(divId, content); - } - return content; + return doc.getDocumentElement().getFirstChild().getTextContent().trim(); } public String unwrap(String v) throws Exception { + if (v == null) { + return null; + } return v.trim().replaceFirst("^<div[^>]*>", "").replaceFirst("</div>$", ""); } private static synchronized void doLogin() throws Exception { if (loginToken == null) { - Document doc = XMLUtils.newDocument(); + Document doc = DOMUtils.createDocument(); Element el = doc.createElementNS(SOAPNS, "ns1:login"); Element el2 = doc.createElement("in0"); @@ -651,85 +787,138 @@ public class SiteExporter implements Run } } - public void loadPagesCache() throws Exception { + public void loadCache() throws Exception { File file = new File(rootOutputDir, pageCacheFile); if (file.exists()) { - FileInputStream fin = new FileInputStream(file); - ObjectInputStream oin = new ObjectInputStream(fin); - pages = CastUtils.cast((Map<?, ?>)oin.readObject()); - oin.close(); + try { + FileInputStream fin = new FileInputStream(file); + ObjectInputStream oin = new ObjectInputStream(fin); + pages = CastUtils.cast((Map<?, ?>)oin.readObject()); + blog = CastUtils.cast((Map<?, ?>)oin.readObject()); + oin.close(); + + for (Page p : pages.values()) { + p.setExporter(this); + } + } catch (Throwable t) { + //invalid cache, punt + pages.clear(); + blog.clear(); + } } } - private Document getPagesDocument() throws Exception { - return getElementsDocument("ns1:getPages"); - } - - private Document getBlogEntriesDocument() throws Exception { - return getElementsDocument("ns1:getBlogEntries"); - } - - private Document getElementsDocument(String function) throws Exception { + public int getBlogVersion(String pageId) throws Exception { Document doc = XMLUtils.newDocument(); - Element el = doc.createElementNS(SOAPNS, function); + Element el = doc.createElementNS(SOAPNS, "ns1:getBlogEntry"); Element el2 = doc.createElement("in0"); el.appendChild(el2); el2.setTextContent(loginToken); el2 = doc.createElement("in1"); el.appendChild(el2); - el2.setTextContent(spaceKey); + el2.setTextContent(pageId); doc.appendChild(el); doc = getDispatch().invoke(doc); - return doc; - } - - private void loadAndAddPages(List<Future<?>> futures, Set<String> allPages, Set<Page> newPages) throws Exception { - Document doc = getPagesDocument(); - ElementLoader loader = new ElementLoader() - { - public Future<?> loadElement(Element element, Set<String> allPages, Set<Page> newPages) throws Exception - { - return loadPage(element, allPages, newPages); - } - }; - loadAndAddElements(futures, loader, doc, allPages, newPages); - } - - private void loadAndAddBlogEntries(List<Future<?>> futures, Set<String> allPages, Set<Page> newPages) throws Exception { - Document doc = getBlogEntriesDocument(); - ElementLoader loader = new ElementLoader() - { - public Future<?> loadElement(Element element, Set<String> allPages, Set<Page> newPages) throws Exception - { - return loadBlogEntry(element, allPages, newPages); - } - }; - loadAndAddElements(futures, loader, doc, allPages, newPages); + + Node nd = doc.getDocumentElement().getFirstChild(); + + String version = DOMUtils.getChildContent(nd, "version"); + return Integer.parseInt(version); } - private void loadAndAddElements(List<Future<?>> futures, ElementLoader loader, Document doc, - Set<String> allPages, Set<Page> newPages) throws Exception { + public void loadBlog() throws Exception { + Document doc = DOMUtils.createDocument(); + Element el = doc.createElementNS(SOAPNS, "ns1:getBlogEntries"); + Element el2 = doc.createElement("in0"); + el.appendChild(el2); + el2.setTextContent(loginToken); + el2 = doc.createElement("in1"); + el.appendChild(el2); + el2.setTextContent(spaceKey); + doc.appendChild(el); + doc = getDispatch().invoke(doc); + + Map<String, BlogEntrySummary> oldBlog = new ConcurrentHashMap<String, BlogEntrySummary>(blog); + Node nd = doc.getDocumentElement().getFirstChild().getFirstChild(); while (nd != null) { if (nd instanceof Element) { - futures.add(loader.loadElement((Element)nd, allPages, newPages)); + BlogEntrySummary entry = new BlogEntrySummary((Element)nd); + entry.setVersion(getBlogVersion(entry.id)); + BlogEntrySummary oldEntry = blog.put(entry.getId(), entry); + if (oldEntry == null || oldEntry.getVersion() != entry.getVersion()) { + modifiedBlog.add(entry); + } + oldBlog.remove(entry.getId()); } nd = nd.getNextSibling(); } + + for (String id : oldBlog.keySet()) { + //these pages have been deleted + BlogEntrySummary p = blog.remove(id); + File file = new File(outputDir, p.getPath()); + if (file.exists()) { + callSvn("rm", file.getAbsolutePath()); + svnCommitMessage.append("Deleted: " + file.getName() + "\n"); + } + if (file.exists()) { + file.delete(); + } + } + } + + public BlogEntrySummary findBlogEntry(String title) throws Exception { + return (BlogEntrySummary) findByTitle(title, blog.values()); } - private interface ElementLoader { - Future<?> loadElement(Element element, Set<String> allPages, Set<Page> newPages) throws Exception; + public BlogEntrySummary findBlogEntryByURL(String url) throws Exception { + return (BlogEntrySummary) findByURL(url, blog.values()); + } + + private static AbstractPage findByURL(String url, Collection<? extends AbstractPage> pages) throws Exception { + for (AbstractPage p : pages) { + if (p.getURL().endsWith(url)) { + return p; + } + } + return null; + } + + private static AbstractPage findByTitle(String title, Collection<? extends AbstractPage> pages) throws Exception { + for (AbstractPage p : pages) { + if (title.equals(p.getTitle())) { + return p; + } + } + return null; } public void loadPages() throws Exception { + Document doc = XMLUtils.newDocument(); + Element el = doc.createElementNS(SOAPNS, "ns1:getPages"); + Element el2 = doc.createElement("in0"); + el.appendChild(el2); + el2.setTextContent(loginToken); + el2 = doc.createElement("in1"); + el.appendChild(el2); + el2.setTextContent(spaceKey); + doc.appendChild(el); + doc = getDispatch().invoke(doc); + Set<String> allPages = new CopyOnWriteArraySet<String>(pages.keySet()); Set<Page> newPages = new CopyOnWriteArraySet<Page>(); List<Future<?>> futures = new ArrayList<Future<?>>(allPages.size()); - - loadAndAddPages(futures, allPages, newPages); - loadAndAddBlogEntries(futures, allPages, newPages); + // XMLUtils.printDOM(doc.getDocumentElement()); + + Node nd = doc.getDocumentElement().getFirstChild().getFirstChild(); + while (nd != null) { + if (nd instanceof Element) { + futures.add(loadPage((Element)nd, allPages, newPages)); + } + nd = nd.getNextSibling(); + } for (Future<?> f : futures) { //wait for all the pages to be done f.get(); @@ -755,6 +944,7 @@ public class SiteExporter implements Run while (checkIncludes()) { // nothing } + } public boolean checkIncludes() { @@ -777,8 +967,7 @@ public class SiteExporter implements Run return false; } public void checkForChildren(Page p) { - String parentId = p.getParentId(); - Page parent = parentId == null ? null : pages.get(parentId); + Page parent = pages.get(p.getParentId()); int d = 1; while (parent != null) { for (Page p2 : pages.values()) { @@ -792,6 +981,32 @@ public class SiteExporter implements Run } } + public static synchronized Space getSpace(String key) { + Space space = spaces.get(key); + if (space == null) { + try { + doLogin(); + + Document doc = XMLUtils.newDocument(); + Element el = doc.createElementNS(SOAPNS, "ns1:getSpace"); + Element el2 = doc.createElement("in0"); + el.appendChild(el2); + el2.setTextContent(loginToken); + el2 = doc.createElement("in1"); + el.appendChild(el2); + el2.setTextContent(key); + doc.appendChild(el); + + Document out = getDispatch().invoke(doc); + space = new Space(out); + spaces.put(key, space); + } catch (Exception e) { + e.printStackTrace(); + } + } + return space; + } + public Future<?> loadPage(Element pageSumEl, final Set<String> allPages, final Set<Page> newPages) throws Exception { @@ -805,45 +1020,20 @@ public class SiteExporter implements Run el2.setTextContent(DOMUtils.getChildContent(pageSumEl, "id")); doc.appendChild(el); - return getResponseHandler(doc, allPages, newPages, false); - } - - public Future<?> loadBlogEntry(Element pageSumEl, final Set<String> allPages, - final Set<Page> newPages) throws Exception - { - Document doc = XMLUtils.newDocument(); - Element el = doc.createElementNS(SOAPNS, "ns1:getBlogEntry"); - Element el2 = doc.createElement("in0"); - el.appendChild(el2); - el2.setTextContent(loginToken); - el2 = doc.createElement("in1"); - el.appendChild(el2); - el2.setTextContent(DOMUtils.getChildContent(pageSumEl, "id")); - doc.appendChild(el); - - return getResponseHandler(doc, allPages, newPages, true); - } - - private Future<?> getResponseHandler(Document doc, final Set<String> allPages, final Set<Page> newPages, final boolean blogPost) throws Exception - { - // make sure we only fire off about 15-20 or confluence may get a bit overloaded - while (asyncCount.get() > 15) - { + //make sure we only fire off about 15-20 or confluence may get a bit overloaded + while (asyncCount.get() > 15) { Thread.sleep(10); } asyncCount.incrementAndGet(); - return getDispatch().invokeAsync(doc, new AsyncHandler<Document>() { + Future<?> f = getDispatch().invokeAsync(doc, new AsyncHandler<Document>() { public void handleResponse(Response<Document> doc) { try { - Page page = new Page(doc.get()); + Page page = new Page(doc.get(), SiteExporter.this); + page.setExporter(SiteExporter.this); Page oldPage = pages.put(page.getId(), page); if (oldPage == null || page.getModifiedTime().compare(oldPage.getModifiedTime()) > 0) { - modifiedPages.add(page); - if(blogPost) { - // if we are adding or modifying a blog post, make sure the pages containing them get re-rendered - for (String title : blogPages) { - modifiedPages.add(findPage(title)); - } + if (!modifiedPages.contains(page)) { + modifiedPages.add(page); } if (oldPage == null) { //need to check parents to see if it has a {children} tag so we can re-render @@ -860,9 +1050,10 @@ public class SiteExporter implements Run } } }); - } + return f; + } - private String updateContentLinks(Page page, String content, + private String updateContentLinks(AbstractPage page, String content, String id, String divCls) throws Exception { XMLReader parser = createTagSoupParser(); StringWriter w = new StringWriter(); @@ -871,6 +1062,14 @@ public class SiteExporter implements Run content = w.toString(); content = content.substring("<html><body>".length()); content = content.substring(0, content.lastIndexOf("</body></html>")); + + int idx = content.indexOf('>'); + if (idx != -1 + && content.substring(idx + 1).startsWith("<p></p>")) { + //new confluence tends to stick an empty paragraph at the beginning for some pages (like Banner) + //that causes major formatting issues. Strip it. + content = content.substring(0, idx + 1) + content.substring(idx + 8); + } return content; } protected XMLReader createTagSoupParser() throws Exception { @@ -886,7 +1085,7 @@ public class SiteExporter implements Run return reader; } - protected ContentHandler createContentHandler(final Page page, Writer w, + protected ContentHandler createContentHandler(AbstractPage page, Writer w, String id, String divCls) { XMLWriter xmlWriter = new ConfluenceCleanupWriter(this, w, page, id, divCls); xmlWriter.setOutputProperty(XMLWriter.OMIT_XML_DECLARATION, "yes"); @@ -903,6 +1102,7 @@ public class SiteExporter implements Run ListIterator<String> it = Arrays.asList(args).listIterator(); List<String> files = new ArrayList<String>(); boolean forceAll = false; + int maxThreads = -1; while (it.hasNext()) { String s = it.next(); if ("-debug".equals(s)) { @@ -919,21 +1119,49 @@ public class SiteExporter implements Run svn = true; } else if ("-commit".equals(s)) { commit = true; + } else if ("-maxThreads".equals(s)) { + maxThreads = Integer.parseInt(it.next()); } else if (s != null && s.length() > 0) { files.add(s); } } - List<Thread> threads = new ArrayList<Thread>(files.size()); + + List<SiteExporter> exporters = new ArrayList<SiteExporter>(); for (String file : files) { - Thread t = new Thread(new SiteExporter(file, forceAll)); - threads.add(t); - t.start(); + exporters.add(new SiteExporter(file, forceAll)); } - for (Thread t : threads) { - t.join(); + List<SiteExporter> modified = new ArrayList<SiteExporter>(); + for (SiteExporter exporter : exporters) { + if (exporter.initialize()) { + modified.add(exporter); + } } + // render stuff only if needed + if (!modified.isEmpty()) { + setSiteExporters(exporters); + + if (maxThreads <= 0) { + maxThreads = modified.size(); + } + + ExecutorService executor = Executors.newFixedThreadPool(maxThreads, new ThreadFactory() { + public Thread newThread(Runnable r) { + Thread t = new Thread(r); + t.setDaemon(true); + return t; + } + }); + List<Future<?>> futures = new ArrayList<Future<?>>(modified.size()); + for (SiteExporter exporter : modified) { + futures.add(executor.submit(exporter)); + } + for (Future<?> t : futures) { + t.get(); + } + } + if (commit) { File file = FileUtils.createTempFile("svncommit", "txt"); FileWriter writer = new FileWriter(file); @@ -943,4 +1171,56 @@ public class SiteExporter implements Run svnCommitMessage.setLength(0); } } + + public boolean hasChildren(Page page) { + for (Page p : pages.values()) { + if (p == page) { + continue; + } + if (page.getId().equals(p.getParentId())) { + return true; + } + } + return false; + } + + public List<Page> getChildren(Page page) { + List<Page> children = new ArrayList<Page>(); + for (Page p : pages.values()) { + if (p == page) { + continue; + } + if (page.getId().equals(p.getParentId())) { + children.add(p); + } + } + return children; + } + + public String link(Page page) { + return page.getLink(); + } + + public Space getSpace() { + if (space == null) { + space = getSpace(spaceKey); + } + return space; + } + + private static void setSiteExporters(List<SiteExporter> exporters) { + siteExporters = exporters; + } + + public int getAPIVersion() { + return apiVersion; + } + + public String stripHost(String value) { + if (value.startsWith(HOST)) { + value = value.substring(HOST.length()); + } + return value; + } + }
Added: tapestry/tapestry-site/trunk/src/main/java/org/apache/cxf/cwiki/Space.java URL: http://svn.apache.org/viewvc/tapestry/tapestry-site/trunk/src/main/java/org/apache/cxf/cwiki/Space.java?rev=1525452&view=auto ============================================================================== --- tapestry/tapestry-site/trunk/src/main/java/org/apache/cxf/cwiki/Space.java (added) +++ tapestry/tapestry-site/trunk/src/main/java/org/apache/cxf/cwiki/Space.java Sun Sep 22 21:33:09 2013 @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cxf.cwiki; + +import org.w3c.dom.Document; + +import org.apache.cxf.helpers.DOMUtils; + +/** + * + */ +public class Space { + + final String key; + final String name; + final String url; + + public Space(Document doc) throws Exception { + // org.apache.cxf.helpers.XMLUtils.printDOM(doc.getDocumentElement()); + + key = DOMUtils.getChildContent(doc.getDocumentElement().getFirstChild(), "key"); + name = DOMUtils.getChildContent(doc.getDocumentElement().getFirstChild(), "name"); + url = DOMUtils.getChildContent(doc.getDocumentElement().getFirstChild(), "url"); + } + + public String getKey() { + return key; + } + public String getName() { + return name; + } + public String getURL() { + return url; + } + +} Propchange: tapestry/tapestry-site/trunk/src/main/java/org/apache/cxf/cwiki/Space.java ------------------------------------------------------------------------------ svn:mime-type = text/plain Modified: tapestry/tapestry-site/trunk/template/template.vm URL: http://svn.apache.org/viewvc/tapestry/tapestry-site/trunk/template/template.vm?rev=1525452&r1=1525451&r2=1525452&view=diff ============================================================================== --- tapestry/tapestry-site/trunk/template/template.vm (original) +++ tapestry/tapestry-site/trunk/template/template.vm Sun Sep 22 21:33:09 2013 @@ -28,7 +28,23 @@ #end </title> <link type="text/css" rel="stylesheet" href="/resources/space.css"> + +#if($page.hasCode) + ## Link directly to Apache CXF site's CSS files for SyntaxHighlighter: + <link href='http://cxf.apache.org/resources/highlighter/styles/shCoreCXF.css' rel='stylesheet' type='text/css' /> + <link href='http://cxf.apache.org/resources/highlighter/styles/shThemeCXF.css' rel='stylesheet' type='text/css' /> + <script src='http://cxf.apache.org/resources/highlighter/scripts/shCore.js' type='text/javascript'></script> +#foreach ($hscript in $page.CodeScripts) + <script src='http://cxf.apache.org/resources/highlighter/scripts/$hscript' type='text/javascript'></script> +#end + <script type="text/javascript"> + SyntaxHighlighter.defaults['toolbar'] = false; + SyntaxHighlighter.all(); + </script> +#end + <link href="/styles/style.css" rel="stylesheet" type="text/css"/> + </head> <body> <div class="wrapper bs">
