Added: manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/client/ConfluenceClient.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/client/ConfluenceClient.java?rev=1874903&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/client/ConfluenceClient.java (added) +++ manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/client/ConfluenceClient.java Fri Mar 6 15:29:45 2020 @@ -0,0 +1,861 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.manifoldcf.crawler.connectors.confluence.v6.client; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.StringReader; +import java.nio.charset.Charset; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.config.SocketConfig; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.client.LaxRedirectStrategy; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.protocol.HttpRequestExecutor; +import org.apache.http.util.EntityUtils; +import org.apache.manifoldcf.connectorcommon.common.InterruptibleSocketFactory; +import org.apache.manifoldcf.connectorcommon.interfaces.KeystoreManagerFactory; +import org.apache.manifoldcf.core.interfaces.ManifoldCFException; +import org.apache.manifoldcf.crawler.connectors.confluence.v6.exception.ConfluenceException; +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.Attachment; +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.ConfluenceResource; +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.ConfluenceResponse; +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.ConfluenceRestrictionsResponse; +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.ConfluenceUser; +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.Group; +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.Label; +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.MutableAttachment; +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.MutablePage; +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.Page; +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.Restrictions; +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.Space; +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.User; +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.builder.ConfluenceResourceBuilder; +import org.apache.manifoldcf.crawler.system.Logging; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Optional; +import com.google.common.collect.Lists; + +/** + * <p> + * ConfluenceClient class + * </p> + * <p> + * This class is intended to be used to interact with Confluence REST API + * </p> + * <p> + * There are some methods that make use of the Confluence JSON-RPC 2.0 API, but until all the methods are ported to the new REST API, we + * will have to use them to leverage all the features provided by Confluence + * </p> + * + * @author Julien Massiera & Antonio David Perez Morales; + * + */ +public class ConfluenceClient { + + private static final String VIEW_PERMISSION = "view"; + + private static final String CONTENT_PATH = "/rest/api/content"; + private static final String AUTHORITY_PATH = "/rpc/json-rpc/confluenceservice-v2/"; + private static final String SPACES_PATH = "/rest/api/space"; + private static final String CHILD_PAGE_PATH = "/child/page"; + private static final String USER_PATH = "/rest/api/user"; + private static final String USER_GROUPS_PATH = "/rest/api/user/memberof"; + private static final String READ_RESTRICTIONS_PATH = "/restriction/byOperation/read"; + private static final String EXPANDABLE_PARAMETERS = "expand=body.view,metadata.labels,space,history,version"; + private static final String CHILD_ATTACHMENTS_PATH = "/child/attachment/"; + private static final String LABEL_PATH = "/label"; + + private final Logger logger = LoggerFactory.getLogger(ConfluenceClient.class); + + private final String protocol; + private final Integer port; + private final String host; + private final String path; + private final String username; + private final String password; + + private int socketTimeout = 900000; + private int connectionTimeout = 60000; + + private CloseableHttpClient httpClient; + private HttpClientContext httpContext; + + /** + * <p> + * Creates a new client instance using the given parameters + * </p> + * + * @param protocol + * the protocol + * @param host + * the host + * @param port + * the port + * @param path + * the path to Confluence instance + * @param username + * the username used to make the requests. Null or empty to use anonymous user + * @param password + * the password + * @throws ManifoldCFException + */ + public ConfluenceClient(final String protocol, final String host, final Integer port, final String path, final String username, final String password, final int socketTimeout, + final int connectionTimeout) throws ManifoldCFException { + this.protocol = protocol; + this.host = host; + this.port = port; + this.path = path; + this.username = username; + this.password = password; + this.socketTimeout = socketTimeout; + this.connectionTimeout = connectionTimeout; + + connect(); + } + + /** + * <p> + * Connect methods used to initialize the underlying client + * </p> + * + * @throws ManifoldCFException + */ + private void connect() throws ManifoldCFException { + + final javax.net.ssl.SSLSocketFactory httpsSocketFactory = KeystoreManagerFactory.getTrustingSecureSocketFactory(); + final SSLConnectionSocketFactory myFactory = new SSLConnectionSocketFactory(new InterruptibleSocketFactory(httpsSocketFactory, connectionTimeout), NoopHostnameVerifier.INSTANCE); + + final PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager( + RegistryBuilder.<ConnectionSocketFactory>create().register("http", PlainConnectionSocketFactory.getSocketFactory()).register("https", myFactory).build()); + poolingConnectionManager.setDefaultMaxPerRoute(1); + poolingConnectionManager.setValidateAfterInactivity(2000); + poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom().setTcpNoDelay(true).setSoTimeout(socketTimeout).build()); + + final RequestConfig.Builder requestBuilder = RequestConfig.custom().setCircularRedirectsAllowed(true).setSocketTimeout(socketTimeout).setExpectContinueEnabled(true) + .setConnectTimeout(connectionTimeout).setConnectionRequestTimeout(socketTimeout); + + httpClient = HttpClients.custom().setConnectionManager(poolingConnectionManager).disableAutomaticRetries().setDefaultRequestConfig(requestBuilder.build()) + .setRequestExecutor(new HttpRequestExecutor(socketTimeout)).setRedirectStrategy(new LaxRedirectStrategy()).build(); + + } + + /** + * <p> + * Close the client. No further requests can be done + * </p> + */ + public void close() { + if (httpClient != null) { + try { + httpClient.close(); + } catch (final IOException e) { + logger.debug("Error closing http connection. Reason: {}", e.getMessage()); + e.printStackTrace(); + } + } + } + + /** + * <p> + * Check method used to test if Confluence instance is up and running + * </p> + * + * @return a {@code Boolean} indicating whether the Confluence instance is alive or not + * + * @throws Exception + */ + public boolean check() throws Exception { + HttpResponse response; + try { + if (httpClient == null) { + connect(); + } + + final String url = String.format(Locale.ROOT, "%s://%s:%s%s%s?limit=1", protocol, host, port, path, CONTENT_PATH); + logger.debug("[Processing] Hitting url: {} for confluence status check fetching : ", "Confluence URL", sanitizeUrl(url)); + final HttpGet httpGet = createGetRequest(url); + response = httpClient.execute(httpGet); + final int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode != 200) { + throw new Exception("[Checking connection] Confluence server appears to be down"); + } else { + return true; + } + } catch (final IOException e) { + logger.warn("[Checking connection] Confluence server appears to be down", e); + throw new Exception("Confluence appears to be down", e); + } + } + + /** + * <p> + * Check method used to test if Confluence instance is up and running when using Authority connector (JSON-RPC API) + * </p> + * <p> + * This method will be deleted when all JSON-RPC methods are available through the REST API + * + * @return a {@code Boolean} indicating whether the Confluence instance is alive or not + * + * @throws Exception + */ + public boolean checkAuth() throws Exception { + try { + if (httpClient == null) { + connect(); + } + getSpaces(); + return true; + } catch (final Exception e) { + logger.warn("[Checking connection] Confluence server appears to be down", e); + throw e; + } + } + + /** + * <p> + * Create a get request for the given url + * </p> + * + * @param url + * the url + * @return the created {@code HttpGet} instance + */ + private HttpGet createGetRequest(final String url) { + final String finalUrl = useBasicAuthentication() ? url + "&os_authType=basic" : url; + final String sanitizedUrl = sanitizeUrl(finalUrl); + final HttpGet httpGet = new HttpGet(sanitizedUrl); + httpGet.addHeader("Accept", "application/json"); + if (useBasicAuthentication()) { + httpGet.addHeader("Authorization", "Basic " + Base64.encodeBase64String(String.format(Locale.ROOT, "%s:%s", this.username, this.password).getBytes(Charset.forName("UTF-8")))); + } + return httpGet; + } + + /** + * + * @param start + * @param limit + * @param space + * @param pageType + * @return + * @throws Exception + */ + @SuppressWarnings("unchecked") + public ConfluenceResponse<Page> getSpaceRootPages(final int start, final int limit, final String space, final Optional<String> pageType) throws Exception { + String contentType = "page"; + if (pageType.isPresent()) { + contentType = pageType.get(); + } + final String url = String.format(Locale.ROOT, "%s://%s:%s%s%s/%s/content/%s?limit=%s&start=%s&depth=root", protocol, host, port, path, SPACES_PATH, space, contentType, limit, start); + return (ConfluenceResponse<Page>) getConfluenceResources(url, Page.builder()); + } + + @SuppressWarnings("unchecked") + public ConfluenceResponse<Page> getPageChilds(final int start, final int limit, final String pageId) throws Exception { + final String url = String.format(Locale.ROOT, "%s://%s:%s%s%s/%s%s?limit=%s&start=%s", protocol, host, port, path, CONTENT_PATH, pageId, CHILD_PAGE_PATH, limit, start); + return (ConfluenceResponse<Page>) getConfluenceResources(url, Page.builder()); + } + + /** + * <p> + * Get a list of Confluence pages using pagination + * </p> + * + * @param start + * The start value to get pages from + * @param limit + * The number of pages to get from start + * @return a {@code ConfluenceResponse} containing the result pages and some pagination values + * @throws Exception + */ + @SuppressWarnings("unchecked") + public ConfluenceResponse<Page> getPages(final int start, final int limit, final Optional<String> space, final Optional<String> pageType) throws Exception { + String url = String.format(Locale.ROOT, "%s://%s:%s%s%s?limit=%s&start=%s", protocol, host, port, path, CONTENT_PATH, limit, start); + if (space.isPresent()) { + url = String.format(Locale.ROOT, "%s&spaceKey=%s", url, space.get()); + } + if (pageType.isPresent()) { + url = String.format(Locale.ROOT, "%s&type=%s", url, pageType.get()); + } + return (ConfluenceResponse<Page>) getConfluenceResources(url, Page.builder()); + } + + /** + * <p> + * Get the {@code ConfluenceResources} from the given url + * </p> + * + * @param url + * The url identifying the REST resource to get the documents + * @param builder + * The builder used to build the resources contained in the response + * @return a {@code ConfluenceResponse} containing the page results + * @throws Exception + */ + private ConfluenceResponse<? extends ConfluenceResource> getConfluenceResources(final String url, final ConfluenceResourceBuilder<? extends ConfluenceResource> builder) throws Exception { + logger.debug("[Processing] Hitting url for get confluence resources: {}", sanitizeUrl(url)); + + final HttpGet httpGet = createGetRequest(url); + try (CloseableHttpResponse response = executeRequest(httpGet);) { + final ConfluenceResponse<? extends ConfluenceResource> confluenceResponse = responseFromHttpEntity(response.getEntity(), builder); + EntityUtils.consume(response.getEntity()); + return confluenceResponse; + } catch (final IOException e) { + logger.error("[Processing] Failed to get page(s)", e); + throw new Exception("Confluence appears to be down", e); + } + } + + /** + * <p> + * Get the {@code ConfluenceResources} from the given url + * </p> + * + * @param url + * The url identifying the REST resource to get the documents + * @param builder + * The builder used to build the resources contained in the response + * @return a {@code ConfluenceRestrictionsResponse} containing the page results + * @throws Exception + */ + private ConfluenceRestrictionsResponse<? extends ConfluenceResource> getConfluenceRestrictionsResources(final String url, final ConfluenceResourceBuilder<? extends ConfluenceResource> builder) + throws Exception { + logger.debug("[Processing] Hitting url for get confluence resources: {}", sanitizeUrl(url)); + + final HttpGet httpGet = createGetRequest(url); + try (CloseableHttpResponse response = executeRequest(httpGet);) { + final ConfluenceRestrictionsResponse<? extends ConfluenceResource> confluenceResponse = restrictionsResponseFromHttpEntity(response.getEntity(), builder); + EntityUtils.consume(response.getEntity()); + return confluenceResponse; + } catch (final IOException e) { + logger.error("[Processing] Failed to get page(s)", e); + throw new Exception("Confluence appears to be down", e); + } + } + + /** + * <p> + * Creates a ConfluenceResponse from the entity returned in the HttpResponse + * </p> + * + * @param entity + * the {@code HttpEntity} to extract the response from + * @return a {@code ConfluenceResponse} with the requested information + * @throws Exception + */ + private <T extends ConfluenceResource> ConfluenceResponse<T> responseFromHttpEntity(final HttpEntity entity, final ConfluenceResourceBuilder<T> builder) throws Exception { + final String stringEntity = EntityUtils.toString(entity, "UTF-8"); + + final JSONParser parser = new JSONParser(); + final JSONObject responseObject = (JSONObject) parser.parse(new StringReader(stringEntity)); + final ConfluenceResponse<T> response = ConfluenceResponse.fromJson(responseObject, builder); + if (response.getResults().size() == 0) { + logger.debug("[Processing] No {} found in the Confluence response", builder.getType().getSimpleName()); + } + + return response; + } + + /** + * <p> + * Creates a ConfluenceResponse from the entity returned in the HttpResponse + * </p> + * + * @param entity + * the {@code HttpEntity} to extract the response from + * @return a {@code ConfluenceResponse} with the requested information + * @throws Exception + */ + private <T extends ConfluenceResource> ConfluenceRestrictionsResponse<T> restrictionsResponseFromHttpEntity(final HttpEntity entity, final ConfluenceResourceBuilder<T> builder) throws Exception { + final String stringEntity = EntityUtils.toString(entity, "UTF-8"); + + final JSONParser parser = new JSONParser(); + final JSONObject responseObject = (JSONObject) parser.parse(new StringReader(stringEntity)); + final ConfluenceRestrictionsResponse<T> response = ConfluenceRestrictionsResponse.fromJson(responseObject, builder); + if (response.getResult() == null) { + logger.debug("[Processing] No {} found in the Confluence Restrictions response", builder.getType().getSimpleName()); + } + + return response; + } + + /** + * <p> + * Get the attachments of the given page + * </p> + * + * @param pageId + * the page id + * @return a {@code ConfluenceResponse} instance containing the attachment results and some pagination values + * @throws Exception + */ + public ConfluenceResponse<Attachment> getPageAttachments(final String pageId) throws Exception { + return getPageAttachments(pageId, 0, 50); + } + + /** + * <p> + * Get the attachments of the given page using pagination + * </p> + * + * @param pageId + * the page id + * @param start + * The start value to get attachments from + * @param limit + * The number of attachments to get from start + * @return a {@code ConfluenceResponse} instance containing the attachment results and some pagination values + * @throws Exception + */ + public ConfluenceResponse<Attachment> getPageAttachments(final String pageId, final int start, final int limit) throws Exception { + final String url = String.format(Locale.ROOT, "%s://%s:%s%s%s/%s%s?limit=%s&start=%s", protocol, host, port, path, CONTENT_PATH, pageId, CHILD_ATTACHMENTS_PATH, limit, start); + @SuppressWarnings("unchecked") + final ConfluenceResponse<Attachment> confluenceResources = (ConfluenceResponse<Attachment>) getConfluenceResources(url, Attachment.builder()); + return confluenceResources; + } + + /** + * <p> + * Gets a specific attachment contained in the specific page + * </p> + * + * @param attachmentId + * @return the {@code Attachment} instance + * @throws Exception + */ + public Attachment getAttachment(final String attachmentId) throws Exception { + final String url = String.format(Locale.ROOT, "%s://%s:%s%s%s/%s?%s", protocol, host, port, path, CONTENT_PATH, attachmentId, EXPANDABLE_PARAMETERS); + logger.debug("[Processing] Hitting url for getting document content : {}", sanitizeUrl(url)); + final HttpGet httpGet = createGetRequest(url); + try (CloseableHttpResponse response = executeRequest(httpGet);) { + final HttpEntity entity = response.getEntity(); + final MutableAttachment attachment = attachmentFromHttpEntity(entity); + EntityUtils.consume(entity); + retrieveAndSetAttachmentContent(attachment); + return attachment; + } catch (final Exception e) { + logger.error("[Processing] Failed to get attachment {}. Error: {}", url, e.getMessage()); + throw e; + } + } + + /** + * <p> + * Downloads and retrieves the attachment content, setting it in the given {@code Attachment} instance + * </p> + * + * @param attachment + * the {@code Attachment} instance to download and set the content + * @throws Exception + */ + private void retrieveAndSetAttachmentContent(final MutableAttachment attachment) throws Exception { + final StringBuilder sb = new StringBuilder(); + sb.append(attachment.getBaseUrl()).append(attachment.getUrlContext()).append(attachment.getDownloadUrl()); + final String url = sanitizeUrl(sb.toString()); + logger.debug("[Processing] Hitting url for getting attachment content : {}", url); + final HttpGet httpGet = createGetRequest(url); + try (CloseableHttpResponse response = executeRequest(httpGet);) { + attachment.setLength(response.getEntity().getContentLength()); + final byte[] byteContent = IOUtils.toByteArray(response.getEntity().getContent()); + EntityUtils.consumeQuietly(response.getEntity()); + attachment.setContentStream(new ByteArrayInputStream(byteContent)); + } catch (final Exception e) { + + logger.error("[Processing] Failed to get attachment content from {}. Error: {}", url, e.getMessage()); + throw e; + } + + } + + /** + * <p> + * Get a Confluence page identified by its id + * </p> + * + * @param pageId + * the page id + * @return the Confluence page + * @throws Exception + */ + public Page getPage(final String pageId) throws Exception { + String url = String.format(Locale.ROOT, "%s://%s:%s%s%s/%s?%s", protocol, host, port, path, CONTENT_PATH, pageId, EXPANDABLE_PARAMETERS); + url = sanitizeUrl(url); + logger.debug("[Processing] Hitting url for getting document content : {}", url); + final HttpGet httpGet = createGetRequest(url); + try (CloseableHttpResponse response = executeRequest(httpGet);) { + final HttpEntity entity = response.getEntity(); + final MutablePage page = pageFromHttpEntity(entity); + EntityUtils.consume(entity); + final List<Label> labels = getLabels(pageId); + page.setLabels(labels); + return page; + } catch (final Exception e) { + logger.error("[Processing] Failed to get page {}. Error: {}", url, e.getMessage()); + throw e; + } + } + + @SuppressWarnings("unchecked") + public ConfluenceRestrictionsResponse<Restrictions> getPageReadRestrictions(final int start, final int limit, final String pageId) throws Exception { + final String url = String.format(Locale.ROOT, "%s://%s:%s%s%s/%s%s?limit=%s&start=%s", protocol, host, port, path, CONTENT_PATH, pageId, READ_RESTRICTIONS_PATH, limit, start); + return (ConfluenceRestrictionsResponse<Restrictions>) getConfluenceRestrictionsResources(url, Restrictions.builder()); + } + + /** + * <p> + * Get the labels of a specific page + * </p> + * + * @param pageId + * The pageId to get the labels + * @return a {@code List<Label>} of labels + */ + public List<Label> getLabels(final String pageId) { + + final List<Label> labels = Lists.newArrayList(); + int lastStart = 0; + final int limit = 50; + boolean isLast = false; + do { + String url = String.format(Locale.ROOT, "%s://%s:%s%s%s/%s/%s?start=%s&limit=%s", protocol, host, port, path, CONTENT_PATH, pageId, LABEL_PATH, lastStart, limit); + url = sanitizeUrl(url); + logger.debug("[Processing] Hitting url for getting page labels : {}", url); + try { + @SuppressWarnings("unchecked") + final ConfluenceResponse<Label> response = (ConfluenceResponse<Label>) getConfluenceResources(url, Label.builder()); + labels.addAll(response.getResults()); + lastStart += response.getResults().size(); + isLast = response.isLast(); + } catch (final Exception e) { + logger.debug("Error getting labels for page {}. Reason: {}", pageId, e.getMessage()); + } + } while (!isLast); + + return labels; + } + + /** + * + * @param username + * @return + * @throws Exception + */ + public ConfluenceUser getUserAuthorities(final String username) throws Exception { + final List<String> authorities = Lists.<String>newArrayList(); + final List<Group> groups = getUserGroups(username); + groups.forEach(group -> { + authorities.add("group-" + group.getName()); + }); + final User user = getConfluenceUser(username); + authorities.add("user-" + user.getUserKey()); + final List<Space> spaces = getSpaces(); + for (final Space space : spaces) { + final List<String> permissions = getSpacePermissionsForUser(space, username); + if (permissions.contains(VIEW_PERMISSION)) { + authorities.add("space-" + space.getKey()); + } + } + return new ConfluenceUser(username, authorities); + + } + + private User getConfluenceUser(final String username) throws Exception { + final String url = String.format(Locale.ROOT, "%s://%s:%s%s%s?username=%s", protocol, host, port, path, USER_PATH, username); + final HttpGet httpGet = createGetRequest(url); + try (CloseableHttpResponse response = executeRequest(httpGet);) { + final HttpEntity entity = response.getEntity(); + final User user = userFromHttpEntity(entity); + EntityUtils.consume(entity); + return user; + } + } + + private List<Group> getUserGroups(final String username) throws Exception { + long lastStart = 0; + final long defaultSize = 50; + final List<Group> groups = new ArrayList<Group>(); + + if (Logging.connectors != null && Logging.connectors.isDebugEnabled()) { + final String groupsDesc = "groups of user " + username; + Logging.connectors.debug(new MessageFormat("Starting from {0} and size {1} for {2}", Locale.ROOT).format(new Object[] { lastStart, defaultSize, groupsDesc })); + } + + Boolean isLast = true; + do { + final ConfluenceResponse<Group> response = getUserGroups((int) lastStart, (int) defaultSize, username); + + int count = 0; + for (final Group group : response.getResults()) { + groups.add(group); + count++; + } + + lastStart += count; + isLast = response.isLast(); + if (Logging.connectors != null && Logging.connectors.isDebugEnabled()) { + Logging.connectors.debug(new MessageFormat("New start {0} and size {1}", Locale.ROOT).format(new Object[] { lastStart, defaultSize })); + } + } while (!isLast); + + return groups; + } + + @SuppressWarnings("unchecked") + public ConfluenceResponse<Group> getUserGroups(final int start, final int limit, final String username) throws Exception { + final String url = String.format(Locale.ROOT, "%s://%s:%s%s%s?username=%s&limit=%s&start=%s", protocol, host, port, path, USER_GROUPS_PATH, username, limit, start); + return (ConfluenceResponse<Group>) getConfluenceResources(url, Group.builder()); + } + + private HttpPost createPostRequest(final String url) { + final HttpPost httpPost = new HttpPost(url); + httpPost.addHeader("Accept", "application/json"); + httpPost.addHeader("Content-Type", "application/json"); + if (useBasicAuthentication()) { + httpPost.addHeader("Authorization", "Basic " + Base64.encodeBase64String(String.format(Locale.ROOT, "%s:%s", this.username, this.password).getBytes(Charset.forName("UTF-8")))); + } + return httpPost; + } + + /** + * <p> + * Execute the given {@code HttpUriRequest} using the configured client + * </p> + * + * @param request + * the {@code HttpUriRequest} to be executed + * @return the {@code HttpResponse} object returned from the server + * @throws Exception + */ + private CloseableHttpResponse executeRequest(final HttpUriRequest request) throws Exception { + final String url = request.getURI().toString(); + logger.debug("[Processing] Hitting url for getting document content : {}", url); + + CloseableHttpResponse response = null; + try { + response = httpClient.execute(request, httpContext); + if (response.getStatusLine().getStatusCode() != 200) { + final String errorDesc = response.getStatusLine().getStatusCode() + " " + response.getStatusLine().getReasonPhrase(); + response.close(); + throw new Exception("Confluence error. " + errorDesc); + } + return response; + } catch (final Exception e) { + if (response != null) { + response.close(); + } + logger.error("[Processing] Failed to get page {}. Error: {}", url, e.getMessage()); + throw e; + } + } + + /** + * <p> + * Creates a Confluence user object from the given entity returned by the server + * </p> + * + * @param entity + * the {@code HttpEntity} to create the {@code User} + * @return the Confluence user instance + * @throws Exception + */ + private User userFromHttpEntity(final HttpEntity entity) throws Exception { + final String stringEntity = EntityUtils.toString(entity, "UTF-8"); + + final JSONParser parser = new JSONParser(); + final JSONObject responseObject = (JSONObject) parser.parse(new StringReader(stringEntity)); + final User user = User.builder().fromJson(responseObject); + return user; + } + + /** + * <p> + * Creates a Confluence page object from the given entity returned by the server + * </p> + * + * @param entity + * the {@code HttpEntity} to create the {@code MutablePage} from + * @return the Confluence page instance + * @throws Exception + */ + private MutablePage pageFromHttpEntity(final HttpEntity entity) throws Exception { + final String stringEntity = EntityUtils.toString(entity, "UTF-8"); + + final JSONParser parser = new JSONParser(); + final JSONObject responseObject = (JSONObject) parser.parse(new StringReader(stringEntity)); + @SuppressWarnings("unchecked") + final MutablePage response = ((ConfluenceResourceBuilder<MutablePage>) MutablePage.builder()).fromJson(responseObject, new MutablePage()); + return response; + } + + /** + * <p> + * Creates a {@code MutableAttachment} object from the given entity returned by the server + * </p> + * + * @param entity + * the {@code HttpEntity} to create the {@code MutableAttachment} from + * @return the Confluence MutableAttachment instance + * @throws Exception + */ + private MutableAttachment attachmentFromHttpEntity(final HttpEntity entity) throws Exception { + final String stringEntity = EntityUtils.toString(entity, "UTF-8"); + final JSONParser parser = new JSONParser(); + final JSONObject responseObject = (JSONObject) parser.parse(new StringReader(stringEntity)); + ; + final MutableAttachment response = (MutableAttachment) Attachment.builder().fromJson(responseObject, new MutableAttachment()); + return response; + } + + /** + * <p> + * Method to check if basic authentication must be used + * </p> + * + * @return {@code Boolean} indicating whether basic authentication must be used or not + */ + private boolean useBasicAuthentication() { + return this.username != null && !"".equals(username) && this.password != null; + } + + /** + * <p> + * Sanitize the given url replacing the appearance of more than one slash by only one slash + * </p> + * + * @param url + * The url to sanitize + * @return the sanitized url + */ + private String sanitizeUrl(final String url) { + final int colonIndex = url.indexOf(":"); + final String urlWithoutProtocol = url.startsWith("http") ? url.substring(colonIndex + 3) : url; + final String sanitizedUrl = urlWithoutProtocol.replaceAll("\\/+", "/"); + return url.substring(0, colonIndex) + "://" + sanitizedUrl; + } + + @SuppressWarnings("unchecked") + public ConfluenceResponse<Space> getSpaces(final int start, final int limit, final Optional<String> spaceType, final Optional<String> spaceStatus) throws Exception { + String url = String.format(Locale.ROOT, "%s://%s:%s%s%s?limit=%s&start=%s", protocol, host, port, path, SPACES_PATH, limit, start); + if (spaceType.isPresent()) { + url = String.format(Locale.ROOT, "%s&type=%s", url, spaceType.get()); + } + if (spaceStatus.isPresent()) { + url = String.format(Locale.ROOT, "%s&status=%s", url, spaceStatus.get()); + } + return (ConfluenceResponse<Space>) getConfluenceResources(url, Space.builder()); + } + + /** + * Get all spaces from Confluence + * + * @return all found spaces + * @throws Exception + */ + private List<Space> getSpaces() throws Exception { + final List<Space> spaces = new ArrayList<>(); + long lastStart = 0; + final long defaultSize = 25; + Boolean isLast = true; + do { + final ConfluenceResponse<Space> response = getSpaces((int) lastStart, (int) defaultSize, Optional.<String>absent(), Optional.<String>absent()); + + spaces.addAll(response.getResults()); + + lastStart += response.getResults().size(); + isLast = response.isLast(); + if (Logging.connectors != null && Logging.connectors.isDebugEnabled()) { + Logging.connectors.debug(new MessageFormat("New start {0} and size {1} for {2}", Locale.ROOT).format(new Object[] { lastStart, defaultSize, "getSpaces" })); + } + } while (!isLast); + return spaces; + } + + private List<String> getSpacePermissionsForUser(final Space space, final String username) throws Exception { + final String url = String.format(Locale.ROOT, "%s://%s:%s%s%sgetPermissionsForUser", protocol, host, port, path, AUTHORITY_PATH); + + logger.debug("[Processing] Hitting url {} for getting Confluence permissions for user {} in space {}", url, username, space.getKey()); + + final HttpPost httpPost = createPostRequest(url); + final JSONArray jsonArray = new JSONArray(); + jsonArray.add(space.getKey()); + jsonArray.add(username); + final StringEntity stringEntity = new StringEntity(jsonArray.toJSONString()); + httpPost.setEntity(stringEntity); + final HttpResponse response = httpClient.execute(httpPost); + if (response.getStatusLine().getStatusCode() != 200) { + throw new ConfluenceException("Confluence error. " + response.getStatusLine().getStatusCode() + " " + response.getStatusLine().getReasonPhrase()); + } + final HttpEntity entity = response.getEntity(); + final List<String> permissions = permissionsFromHttpEntity(entity); + EntityUtils.consume(entity); + return permissions; + } + + private List<String> permissionsFromHttpEntity(final HttpEntity entity) throws Exception { + final String stringEntity = EntityUtils.toString(entity, "UTF-8"); + final JSONParser parser = new JSONParser(); + final Object parsedReponse = parser.parse(new StringReader(stringEntity)); + final List<String> permissions = Lists.newArrayList(); + if (parsedReponse instanceof JSONArray) { + final JSONArray responseObject = (JSONArray) parsedReponse; + for (int i = 0, len = responseObject.size(); i < len; i++) { + permissions.add(responseObject.get(i).toString()); + } + } else { + final JSONObject responseObject = (JSONObject) parsedReponse; + if (responseObject.containsKey("error")) { + final JSONObject error = (JSONObject) responseObject.get("error"); + final String message = error.get("message").toString(); + // Probably has no permissions to get this space's permissions + logger.warn("Confluence authority: Can't get user permissions; " + message); + return new ArrayList<>(0); + } else { + throw new Exception("Unexpected JSON format: " + responseObject); + } + } + + return permissions; + } +}
Added: manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/exception/ConfluenceException.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/exception/ConfluenceException.java?rev=1874903&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/exception/ConfluenceException.java (added) +++ manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/exception/ConfluenceException.java Fri Mar 6 15:29:45 2020 @@ -0,0 +1,34 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.manifoldcf.crawler.connectors.confluence.v6.exception; + +public class ConfluenceException extends Exception { + + /** + * + */ + private static final long serialVersionUID = 5903550079897330304L; + + public ConfluenceException(String message) { + super(message); + } + + public ConfluenceException(String message, Throwable cause) { + super(message, cause); + } +} Added: manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/exception/PageNotFoundException.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/exception/PageNotFoundException.java?rev=1874903&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/exception/PageNotFoundException.java (added) +++ manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/exception/PageNotFoundException.java Fri Mar 6 15:29:45 2020 @@ -0,0 +1,31 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.manifoldcf.crawler.connectors.confluence.v6.exception; + +/** + * + * @author Antonio David Perez Morales <[email protected]> + * + */ +public class PageNotFoundException extends Exception { + + /** + * + */ + private static final long serialVersionUID = 1L; +} \ No newline at end of file Added: manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/Attachment.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/Attachment.java?rev=1874903&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/Attachment.java (added) +++ manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/Attachment.java Fri Mar 6 15:29:45 2020 @@ -0,0 +1,128 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.manifoldcf.crawler.connectors.confluence.v6.model; + +import java.io.InputStream; +import java.util.Map; + +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.builder.ConfluenceResourceBuilder; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; + +/** + * <p> + * Attachment class + * </p> + * <p> + * Represents a Confluence Attachment + * </p> + * + * @author Antonio David Perez Morales <[email protected]> + */ +public class Attachment extends Page { + + protected static final String KEY_DOWNLOAD = "download"; + protected static final String KEY_EXTENSIONS = "extensions"; + protected String downloadUrl; + protected InputStream contentStream; + + public static ConfluenceResourceBuilder<Attachment> builder() { + return new AttachmentBuilder(); + } + + public String getDownloadUrl() { + return this.downloadUrl; + } + + @Override + public boolean hasContent() { + return (this.length > 0 && this.hasContentStream()) || (this.downloadUrl != null && !this.downloadUrl.isEmpty()); + } + + public Boolean hasContentStream() { + return this.contentStream != null; + } + + @Override + public InputStream getContentStream() { + if(hasContentStream()) { + return this.contentStream; + } + return super.getContentStream(); + } + + @Override + protected void refineMetadata(Map<String, Object> metadata) { + super.refineMetadata(metadata); + metadata.put("downloadUrl", this.getBaseUrl() + this.getUrlContext() + + downloadUrl); + } + + /** + * <p> + * AttachmentBuilder internal class + * </p> + * <p> + * Used to build Attachments + * </p> + * + * @author Antonio David Perez Morales <[email protected]> + * + */ + public static class AttachmentBuilder implements ConfluenceResourceBuilder<Attachment>{ + + @Override + public Attachment fromJson(JSONObject jsonPage) { + return fromJson(jsonPage, new Attachment()); + } + + @SuppressWarnings("unchecked") + public Attachment fromJson(JSONObject jsonPage, Attachment attachment) { + ((ConfluenceResourceBuilder<Page>) Page.builder()).fromJson(jsonPage, attachment); + + /* + * Download URL + */ + + JSONObject links = (JSONObject) jsonPage.get(Page.KEY_LINKS); + if (links != null) { + attachment.downloadUrl = (links.get(KEY_DOWNLOAD)==null)?"":links.get(KEY_DOWNLOAD).toString(); + } + + /* + * Extensions + */ + JSONObject extensions = (JSONObject) jsonPage + .get(KEY_EXTENSIONS); + if (extensions != null) { + final Object o = extensions.get(Page.KEY_MEDIATYPE); + attachment.mediaType = (o==null)?"":o.toString(); + } + + return attachment; + } + + @Override + public Class<Attachment> getType() { + return Attachment.class; + } + + } +} Added: manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/ConfluenceResource.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/ConfluenceResource.java?rev=1874903&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/ConfluenceResource.java (added) +++ manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/ConfluenceResource.java Fri Mar 6 15:29:45 2020 @@ -0,0 +1,29 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.manifoldcf.crawler.connectors.confluence.v6.model; + +/** + * <p>ConfluenceResource class</p> + * <p>Used as base class for other classes like Page and Attachments</p> + * + * @author Antonio David Perez Morales <[email protected]> + * + */ +public class ConfluenceResource { + +} Added: manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/ConfluenceResponse.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/ConfluenceResponse.java?rev=1874903&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/ConfluenceResponse.java (added) +++ manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/ConfluenceResponse.java Fri Mar 6 15:29:45 2020 @@ -0,0 +1,76 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.manifoldcf.crawler.connectors.confluence.v6.model; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.builder.ConfluenceResourceBuilder; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; + +public class ConfluenceResponse<T extends ConfluenceResource> { + + private List<T> results; + private int start; + private int limit; + private Boolean isLast; + + public ConfluenceResponse(List<T> results, int start, int limit, Boolean isLast) { + this.results = results; + this.start = start; + this.limit = limit; + this.isLast = isLast; + } + + public List<T> getResults() { + return this.results; + } + + public int getStart() { + return this.start; + } + + public int getLimit() { + return this.limit; + } + + public Boolean isLast() { + return isLast; + } + + public static <T extends ConfluenceResource> ConfluenceResponse<T> fromJson(JSONObject response, ConfluenceResourceBuilder<T> builder) { + List<T> resources = new ArrayList<T>(); + JSONArray jsonArray = (JSONArray)response.get("results"); + for(int i=0,size=jsonArray.size(); i<size;i++) { + JSONObject jsonPage = (JSONObject)jsonArray.get(i); + T resource = (T) builder.fromJson(jsonPage); + resources.add(resource); + } + + int limit = ((Long)response.get("limit")).intValue(); + int start = ((Long)response.get("start")).intValue(); + Boolean isLast = false; + JSONObject links = (JSONObject)response.get("_links"); + if(links != null) { + isLast = (links.get("next")==null); + } + + return new ConfluenceResponse<T>(resources, start, limit, isLast); + } +} Added: manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/ConfluenceRestrictionsResponse.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/ConfluenceRestrictionsResponse.java?rev=1874903&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/ConfluenceRestrictionsResponse.java (added) +++ manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/ConfluenceRestrictionsResponse.java Fri Mar 6 15:29:45 2020 @@ -0,0 +1,102 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.manifoldcf.crawler.connectors.confluence.v6.model; + +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.builder.ConfluenceResourceBuilder; +import org.json.simple.JSONObject; + +public class ConfluenceRestrictionsResponse<T extends ConfluenceResource> { + + private final T result; + private final int start; + private final int limit; + private final Boolean isLast; + + public ConfluenceRestrictionsResponse(final T result, final int start, final int limit, final Boolean isLast) { + this.result = result; + this.start = start; + this.limit = limit; + this.isLast = isLast; + } + + public T getResult() { + return this.result; + } + + public int getStart() { + return this.start; + } + + public int getLimit() { + return this.limit; + } + + public Boolean isLast() { + return isLast; + } + + public static <T extends ConfluenceResource> ConfluenceRestrictionsResponse<T> fromJson(final JSONObject response, final ConfluenceResourceBuilder<T> builder) { + final JSONObject restrictions = (JSONObject) response.get("restrictions"); + JSONObject restrictionsUser = new JSONObject(); + JSONObject restrictionsGroup = new JSONObject(); + if (restrictions.get("user") != null) { + restrictionsUser = (JSONObject) restrictions.get("user"); + } + if (restrictions.get("group") != null) { + restrictionsGroup = (JSONObject) restrictions.get("group"); + } + + final T resource = builder.fromJson(restrictions); + + Boolean isLast = false; + Boolean isLastUser = false; + Boolean isLastGroup = false; + int userLimit = -1; + int groupLimit = -1; + int userStart = -1; + int groupStart = -1; + if (restrictionsUser.get("limit") != null) { + userLimit = ((Long) restrictionsUser.get("limit")).intValue(); + userStart = ((Long) restrictionsUser.get("start")).intValue(); + final int userSize = ((Long) restrictionsUser.get("size")).intValue(); + if (userSize < userLimit) { + isLastUser = true; + } + } else { + isLastUser = true; + } + + if (restrictionsGroup.get("limit") != null) { + groupLimit = ((Long) restrictionsGroup.get("limit")).intValue(); + groupStart = ((Long) restrictionsGroup.get("start")).intValue(); + final int groupSize = ((Long) restrictionsGroup.get("size")).intValue(); + if (groupSize < groupLimit) { + isLastGroup = true; + } + } else { + isLastGroup = true; + } + + isLast = isLastUser && isLastGroup; + + final int limit = userLimit > -1 ? userLimit : groupLimit; + final int start = userStart > -1 ? userStart : groupStart; + + return new ConfluenceRestrictionsResponse<T>(resource, start, limit, isLast); + } +} Added: manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/ConfluenceUser.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/ConfluenceUser.java?rev=1874903&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/ConfluenceUser.java (added) +++ manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/ConfluenceUser.java Fri Mar 6 15:29:45 2020 @@ -0,0 +1,45 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.manifoldcf.crawler.connectors.confluence.v6.model; + +import java.util.List; + +/** + * <p>ConfluenceUser class</p> + * <p>Represents a Confluence user</p> + * + * @author Antonio David Perez Morales <[email protected]> + * + */ +public class ConfluenceUser { + private final String username; + private final List<String> authorities; + + public ConfluenceUser(String username, List<String> authorities) { + this.username = username; + this.authorities = authorities; + } + + public String getUsername() { + return username; + } + + public List<String> getAuthorities() { + return authorities; + } + } Added: manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/Group.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/Group.java?rev=1874903&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/Group.java (added) +++ manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/Group.java Fri Mar 6 15:29:45 2020 @@ -0,0 +1,64 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.manifoldcf.crawler.connectors.confluence.v6.model; + +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.builder.ConfluenceResourceBuilder; +import org.json.simple.JSONObject; + +public class Group extends ConfluenceResource { + + protected static final String KEY_NAME = "name"; + protected static final String KEY_TYPE = "type"; + + protected String type; + protected String name; + + public String getType() { + return type; + } + + public String getName() { + return name; + } + + public static ConfluenceResourceBuilder<? extends Group> builder() { + return new GroupBuilder(); + } + + public static class GroupBuilder implements ConfluenceResourceBuilder<Group>{ + + @Override + public Group fromJson(JSONObject groupJson) { + return fromJson(groupJson, new Group()); + } + + @Override + public Group fromJson(JSONObject groupJson, Group group) { + group.type = (groupJson.get(KEY_TYPE)==null)?"":groupJson.get(KEY_TYPE).toString(); + group.name = (groupJson.get(KEY_NAME)==null)?"":groupJson.get(KEY_NAME).toString(); + return group; + } + + @Override + public Class<Group> getType() { + return Group.class; + } + + } + +} Added: manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/Label.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/Label.java?rev=1874903&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/Label.java (added) +++ manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/Label.java Fri Mar 6 15:29:45 2020 @@ -0,0 +1,103 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.manifoldcf.crawler.connectors.confluence.v6.model; + +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.builder.ConfluenceResourceBuilder; +import org.json.simple.JSONObject; + +/** + * <p> + * Label class + * </p> + * <p> + * Represents a Confluence Label + * </p> + * + * @author Antonio David Perez Morales <[email protected]> + */ +public class Label extends ConfluenceResource{ + + protected static final String KEY_LINKS = "_links"; + protected static final String KEY_ID = "id"; + protected static final String KEY_SELF = "self"; + protected static final String KEY_PREFIX = "prefix"; + protected static final String KEY_NAME = "name"; + + protected String id; + protected String prefix; + protected String name; + + @SuppressWarnings("unused") + private JSONObject delegated; + + public Label() { + + } + + public String getId() { + return this.id; + } + + public String getPrefix() { + return this.prefix; + } + + public String getName() { + return this.name; + } + + public static LabelBuilder builder() { + return new LabelBuilder(); + } + + /** + * <p> + * LabelBuilder internal class + * </p> + * <p> + * Used to build Labels + * </p> + * + * @author Antonio David Perez Morales <[email protected]> + * + */ + public static class LabelBuilder implements ConfluenceResourceBuilder<Label>{ + + public Label fromJson(JSONObject jsonLabel) { + return fromJson(jsonLabel, new Label()); + } + + public Label fromJson(JSONObject jsonPage, Label label) { + + label.id = (jsonPage.get(KEY_ID)==null)?"":jsonPage.get(KEY_ID).toString(); + label.prefix = (jsonPage.get(KEY_PREFIX)==null)?"":jsonPage.get(KEY_PREFIX).toString(); + label.name = (jsonPage.get(KEY_NAME)==null)?"":jsonPage.get(KEY_NAME).toString(); + + label.delegated = jsonPage; + + return label; + + } + + @Override + public Class<Label> getType() { + return Label.class; + } + + } +} Added: manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/MutableAttachment.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/MutableAttachment.java?rev=1874903&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/MutableAttachment.java (added) +++ manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/MutableAttachment.java Fri Mar 6 15:29:45 2020 @@ -0,0 +1,111 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.manifoldcf.crawler.connectors.confluence.v6.model; + +import java.io.InputStream; +import java.util.Date; + +/** + * <p> + * Mutable Attachment class + * </p> + * <p> + * Represents a Confluence Attachment which can be mutated + * </p> + * + * @author Antonio David Perez Morales <[email protected]> + */ +public class MutableAttachment extends Attachment { + + public void setId(String id) { + this.id = id; + } + + public void setSpace(String space) { + this.space = space; + } + + public void setBaseUrl(String baseUrl) { + this.baseUrl = baseUrl; + } + + public void setUrlContext(String urlContext) { + this.urlContext = urlContext; + } + + public void setUrl(String url) { + this.url = url; + } + + public void setWebUrl(String webUrl) { + this.webUrl = webUrl; + } + + public void setCreatedDate(Date createdDate) { + this.createdDate = createdDate; + } + + public void setLastModified(Date lastModified) { + this.lastModified = lastModified; + } + + public void setType(PageType type) { + this.type = type; + } + + public void setTitle(String title) { + this.title = title; + } + + public void setVersion(int version) { + this.version = version; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public void setCreatorUsername(String creatorUsername) { + this.creatorUsername = creatorUsername; + } + + public void setLastModifier(String lastModifier) { + this.lastModifier = lastModifier; + } + + public void setLastModifierUsername(String lastModifierUsername) { + this.lastModifierUsername = lastModifierUsername; + } + + public void setMediaType(String mediaType) { + this.mediaType = mediaType; + } + + public void setLength(long length) { + this.length = length; + } + + public void setDownloadUrl(String downloadUrl) { + this.downloadUrl = downloadUrl; + } + + public void setContentStream(InputStream contentStream) { + this.contentStream = contentStream; + } + +} Added: manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/MutablePage.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/MutablePage.java?rev=1874903&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/MutablePage.java (added) +++ manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/MutablePage.java Fri Mar 6 15:29:45 2020 @@ -0,0 +1,116 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.manifoldcf.crawler.connectors.confluence.v6.model; + +import java.util.Date; +import java.util.List; + +/** + * <p> + * MutablePage class + * </p> + * <p> + * Represents a Confluence Page which is mutable unlike {@code Page} class which can be also initialized using the PageBuilder obtained from + * <code>Page.builder()</code> method + * </p> + * + * @author Antonio David Perez Morales <[email protected]> + */ +public class MutablePage extends Page { + + public MutablePage() { + + } + + public void setId(String id) { + this.id = id; + } + + public void setSpace(String space) { + this.space = space; + } + + public void setBaseUrl(String baseUrl) { + this.baseUrl = baseUrl; + } + + public void setUrlContext(String urlContext) { + this.urlContext = urlContext; + } + + public void setUrl(String url) { + this.url = url; + } + + public void setWebUrl(String webUrl) { + this.webUrl = webUrl; + } + + public void setCreatedDate(Date createdDate) { + this.createdDate = createdDate; + } + + public void setLastModified(Date lastModified) { + this.lastModified = lastModified; + } + + public void setType(PageType type) { + this.type = type; + } + + public void setTitle(String title) { + this.title = title; + } + + public void setVersion(int version) { + this.version = version; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public void setCreatorUsername(String creatorUsername) { + this.creatorUsername = creatorUsername; + } + + public void setLastModifier(String lastModifier) { + this.lastModifier = lastModifier; + } + + public void setLastModifierUsername(String lastModifierUsername) { + this.lastModifierUsername = lastModifierUsername; + } + + public void setMediaType(String mediaType) { + this.mediaType = mediaType; + } + + public void setLength(long length) { + this.length = length; + } + + public void setContent(String content) { + this.content = content; + } + + public void setLabels(List<Label> labels) { + this.labels = labels; + } + +} Added: manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/Page.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/Page.java?rev=1874903&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/Page.java (added) +++ manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/Page.java Fri Mar 6 15:29:45 2020 @@ -0,0 +1,353 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.manifoldcf.crawler.connectors.confluence.v6.model; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Date; +import java.util.List; +import java.util.Map; + +import org.apache.manifoldcf.core.common.DateParser; +import org.apache.manifoldcf.crawler.connectors.confluence.v6.model.builder.ConfluenceResourceBuilder; +import org.json.simple.JSONObject; + +import com.google.common.base.Function; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +/** + * <p> + * Page class + * </p> + * <p> + * Represents a Confluence Page + * </p> + * + * @author Antonio David Perez Morales <[email protected]> + */ +public class Page extends ConfluenceResource{ + + protected static final String KEY_LINKS = "_links"; + protected static final String KEY_ID = "id"; + protected static final String KEY_SELF = "self"; + protected static final String KEY_WEBUI = "webui"; + protected static final String KEY_BASE = "base"; + protected static final String KEY_CONTEXT = "context"; + protected static final String KEY_KEY = "key"; + protected static final String KEY_TITLE = "title"; + protected static final String KEY_BODY = "body"; + protected static final String KEY_VIEW = "view"; + protected static final String KEY_VALUE = "value"; + protected static final String KEY_SPACE = "space"; + protected static final String KEY_HISTORY = "history"; + protected static final String KEY_CREATED_DATE = "createdDate"; + protected static final String KEY_CREATED_BY = "createdBy"; + protected static final String KEY_BY = "by"; + protected static final String KEY_TYPE = "type"; + protected static final String KEY_DISPLAY_NAME = "displayName"; + protected static final String KEY_USER_NAME = "username"; + protected static final String KEY_VERSION = "version"; + protected static final String KEY_WHEN = "when"; + protected static final String KEY_MEDIATYPE = "mediaType"; + + private static final String PAGE_ID = "confluenceId"; + private static final String PAGE_URL = "url"; + private static final String PAGE_WEBURL = "webUrl"; + private static final String PAGE_LAST_MODIFIED = "lastModified"; + private static final String PAGE_CREATOR = "creator"; + private static final String PAGE_CREATOR_USERNAME = "creatorUsername"; + private static final String PAGE_LAST_MODIFIER = "lastModifier"; + private static final String PAGE_LAST_MODIFIER_USERNAME = "lastModifierUsername"; + private static final String PAGE_SIZE = "size"; + private static final String PAGE_LABEL = "label"; + + protected String id; + protected String space; + protected String baseUrl; + protected String urlContext; + protected String url; + protected String webUrl; + protected Date createdDate; + protected Date lastModified; + protected PageType type; + protected String title; + protected int version; + protected String creator; + protected String creatorUsername; + protected String lastModifier; + protected String lastModifierUsername; + protected String mediaType = "text/html; charset=utf-8"; + protected long length; + protected String content; + protected List<Label> labels = Lists.newArrayList(); + + @SuppressWarnings("unused") + private JSONObject delegated; + + public Page() { + + } + + public String getContent() { + return this.content; + } + + public String getId() { + return this.id; + } + + public PageType getType() { + return this.type; + } + + public String getMediaType() { + return this.mediaType; + } + + public int getVersion() { + return this.version; + } + + public String getTitle() { + return this.title; + } + + public String getBaseUrl() { + return this.baseUrl; + } + + public String getUrlContext() { + return this.urlContext; + } + + public String getWebUrl() { + return this.webUrl; + } + + public String getUrl() { + return this.url; + } + + public String getSpace() { + return this.space; + } + + public String getCreator() { + return this.creator; + } + + public String getCreatorUsername() { + return this.creatorUsername; + } + + public String getLastModifier() { + return this.lastModifier; + } + + public String getLastModifierUsername() { + return this.lastModifierUsername; + } + + public Date getCreatedDate() { + return this.createdDate; + } + + public Date getLastModifiedDate() { + return this.lastModified; + } + + public long getLength() { + return this.length; + } + + public boolean hasContent() { + return this.length > 0 && this.content != null; + } + + public InputStream getContentStream() { + String contentStream = content != null ? content : ""; + return new ByteArrayInputStream( + contentStream.getBytes(StandardCharsets.UTF_8)); + } + + public List<Label> getLabels() { + return this.labels; + } + + public Map<String, Object> getMetadataAsMap() { + Map<String, Object> pageMetadata = Maps.newHashMap(); + pageMetadata.put(KEY_ID, this.id); + pageMetadata.put(PAGE_ID, this.id); + pageMetadata.put(KEY_TYPE, this.type.toString()); + pageMetadata.put(KEY_TITLE, this.title); + pageMetadata.put(KEY_SPACE, this.space); + pageMetadata.put(PAGE_URL, this.url); + pageMetadata.put(PAGE_WEBURL, this.webUrl); + pageMetadata.put(KEY_CREATED_DATE, + DateParser.formatISO8601Date(this.createdDate)); + pageMetadata.put(PAGE_LAST_MODIFIED, + DateParser.formatISO8601Date(this.lastModified)); + pageMetadata.put(KEY_MEDIATYPE, this.mediaType); + pageMetadata.put(KEY_VERSION, String.valueOf(this.version)); + pageMetadata.put(PAGE_CREATOR, this.creator); + pageMetadata.put(PAGE_CREATOR_USERNAME, this.creatorUsername); + pageMetadata.put(PAGE_LAST_MODIFIER, this.lastModifier); + pageMetadata + .put(PAGE_LAST_MODIFIER_USERNAME, this.lastModifierUsername); + pageMetadata.put(PAGE_SIZE, String.valueOf(this.length)); + + putLabelsOnMetadataMap(pageMetadata); + refineMetadata(pageMetadata); + return pageMetadata; + } + + /** + * <p>Put the page labels on the metadata map</p> + * @param pageMetadata + */ + private void putLabelsOnMetadataMap(Map<String, Object> pageMetadata) { + if(this.labels == null || this.labels.isEmpty()) { + return; + } + + Iterable<String> labelsString = Iterables.transform(this.labels, new Function<Label, String>() { + @Override + public String apply(Label input) { + return input.getName(); + } + }); + + pageMetadata.put(PAGE_LABEL, Lists.newArrayList(labelsString)); + + } + + /** + * <p> + * Used to be overwritten by child classes to add more metadata to the map + * </p> + * + * @param metadata + */ + protected void refineMetadata(Map<String, Object> metadata) { + } + + public static ConfluenceResourceBuilder<? extends Page> builder() { + return new PageBuilder(); + } + + /** + * <p>PageBuilder internal class</p> + * <p>Used to build pages</p> + * @author Antonio David Perez Morales <[email protected]> + * + */ + public static class PageBuilder implements ConfluenceResourceBuilder<Page>{ + + public Page fromJson(JSONObject jsonPage) { + return fromJson(jsonPage, new Page()); + } + + public Page fromJson(JSONObject jsonPage, Page page) { + + String id = jsonPage.get(KEY_ID).toString(); + String type = jsonPage.get(KEY_TYPE).toString(); + String title = jsonPage.get(KEY_TITLE).toString(); + + page.delegated = jsonPage; + + /* Init Page fields */ + page.id = id; + page.type = PageType.fromName(type); + page.title = title; + + page.space = processSpace(jsonPage); + + /* + * Url & WebUrl + */ + JSONObject links = (JSONObject) jsonPage.get(KEY_LINKS); + if (links != null) { + page.url = (links.get(KEY_SELF)==null)?"":links.get(KEY_SELF).toString(); + String webUrl = (links.get(KEY_WEBUI)==null)?"":links.get(KEY_WEBUI).toString(); + page.urlContext = (links.get(KEY_CONTEXT)==null)?"":links.get(KEY_CONTEXT).toString(); + page.baseUrl = (links.get(KEY_BASE)==null)?"":links.get(KEY_BASE).toString(); + page.webUrl = page.baseUrl + webUrl; + + } + + /* + * Created By and created Date + */ + JSONObject history = (JSONObject) jsonPage.get(KEY_HISTORY); + if (history != null) { + + page.createdDate = DateParser.parseISO8601Date((history.get(KEY_CREATED_DATE)==null)?"":history.get(KEY_CREATED_DATE).toString()); + JSONObject createdBy = (JSONObject) history.get(KEY_CREATED_BY); + if (createdBy != null) { + page.creator = (createdBy.get(KEY_DISPLAY_NAME)==null)?"":createdBy.get(KEY_DISPLAY_NAME).toString(); + page.creatorUsername = (createdBy.get(KEY_USER_NAME)==null)?"":createdBy.get(KEY_USER_NAME).toString(); + } + + } + + /* + * Last modifier and Last modified date + */ + JSONObject version = (JSONObject) jsonPage.get(KEY_VERSION); + if (version != null) { + JSONObject by = (JSONObject) version.get(KEY_BY); + if (by != null) { + page.lastModifier = (by.get(KEY_DISPLAY_NAME)==null)?"":by.get(KEY_DISPLAY_NAME).toString(); + page.lastModifierUsername = (by.get(KEY_USER_NAME)==null)?"":by.get(KEY_USER_NAME).toString(); + } + + page.lastModified = DateParser.parseISO8601Date((version.get(KEY_WHEN)==null)?"":version.get(KEY_WHEN).toString()); + } + + /* + * Page Content + */ + JSONObject body = (JSONObject) jsonPage.get(KEY_BODY); + if (body != null) { + JSONObject view = (JSONObject) body.get(KEY_VIEW); + if (view != null) { + page.content = (view.get(KEY_VALUE)==null)?"":view.get(KEY_VALUE).toString(); + page.length = page.content.getBytes(StandardCharsets.UTF_8).length; + } + } + + return page; + } + + private static String processSpace(JSONObject page) { + /* Page */ + JSONObject space = (JSONObject) page.get(KEY_SPACE); + if (space != null) + return (space.get(KEY_KEY)==null)?"":space.get(KEY_KEY).toString(); + return ""; + } + + @Override + public Class<Page> getType() { + return Page.class; + } + } +} Added: manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/PageType.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/PageType.java?rev=1874903&view=auto ============================================================================== --- manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/PageType.java (added) +++ manifoldcf/branches/CONNECTORS-1637/connectors/confluence-v6/src/main/java/org/apache/manifoldcf/crawler/connectors/confluence/v6/model/PageType.java Fri Mar 6 15:29:45 2020 @@ -0,0 +1,48 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.manifoldcf.crawler.connectors.confluence.v6.model; + +import org.apache.commons.lang.WordUtils; + +import java.util.Locale; + +/** + * <p>PageType class</p> + * <p>Represents the kind of pages we can have in Confluence</p> + * + * @author Antonio David Perez Morales <[email protected]> + * + */ +public enum PageType { + + PAGE, ATTACHMENT, BLOGPOST; + + public static PageType fromName(String type) { + for(PageType pageType: values()) { + if(pageType.name().equalsIgnoreCase(type)) { + return pageType; + } + } + + return PageType.PAGE; + } + + public String toString() { + return WordUtils.capitalize(name().toLowerCase(Locale.ROOT)); + } +}
