This is an automated email from the ASF dual-hosted git repository.
piergiorgio pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/manifoldcf.git
The following commit(s) were added to refs/heads/trunk by this push:
new 32a56e198 Merged (manually) pull request #142 by Douglas
32a56e198 is described below
commit 32a56e198b896eb600b2a6e5051205f1fb553b1b
Author: Piergiorgio Lucidi <[email protected]>
AuthorDate: Thu Nov 28 13:54:52 2024 +0100
Merged (manually) pull request #142 by Douglas
---
build.xml | 35 +++++-
connectors/googledrive/build.xml | 126 +++++++++++----------
.../GoogleDriveRepositoryConnector.java | 70 +++++-------
.../connectors/googledrive/GoogleDriveSession.java | 15 ++-
connectors/googledrive/pom.xml | 8 +-
5 files changed, 141 insertions(+), 113 deletions(-)
diff --git a/build.xml b/build.xml
index 3237568a0..f98a36762 100644
--- a/build.xml
+++ b/build.xml
@@ -116,8 +116,12 @@
<property name="dropbox-client.version" value="1.5.3"/>
<property name="json-simple.version" value="1.1.1"/>
<property name="json.version" value="1.8"/>
- <property name="google-api-client.version" value="v2-rev326-1.25.0"/>
- <property name="google-http-client.version" value="1.25.0"/>
+ <property name="google-drive-api-client.version"
value="v3-rev20220815-2.0.0"/>
+ <property name="google-http-client.version" value="1.42.1"/>
+ <property name="google-oauth-client.version" value="1.34.1"/>
+ <property name="google-api-client.version" value="2.0.0"/>
+ <property name="opencensus.version" value="0.31.1"/>
+ <property name="grpc.version" value="1.27.2"/>
<property name="findbugs.version" value="3.0.2"/>
<property name="j2objc.version" value="1.1"/>
<property name="jackson-core.version" value="2.13.1"/>
@@ -2254,7 +2258,7 @@ Use Apache Forrest version forrest-0.9-dev or higher.
<antcall target="download-via-maven">
<param name="target" value="lib"/>
<param name="project-path" value="com/google/apis"/>
- <param name="artifact-version"
value="${google-api-client.version}"/>
+ <param name="artifact-version"
value="${google-drive-api-client.version}"/>
<param name="artifact-name" value="google-api-services-drive"/>
<param name="artifact-type" value="jar"/>
</antcall>
@@ -2289,7 +2293,7 @@ Use Apache Forrest version forrest-0.9-dev or higher.
<antcall target="download-via-maven">
<param name="target" value="lib"/>
<param name="project-path" value="com/google/oauth-client"/>
- <param name="artifact-version"
value="${google-http-client.version}"/>
+ <param name="artifact-version"
value="${google-oauth-client.version}"/>
<param name="artifact-name" value="google-oauth-client"/>
<param name="artifact-type" value="jar"/>
</antcall>
@@ -2303,10 +2307,31 @@ Use Apache Forrest version forrest-0.9-dev or higher.
<antcall target="download-via-maven">
<param name="target" value="lib"/>
<param name="project-path" value="com/google/api-client"/>
- <param name="artifact-version"
value="${google-http-client.version}"/>
+ <param name="artifact-version"
value="${google-api-client.version}"/>
<param name="artifact-name" value="google-api-client"/>
<param name="artifact-type" value="jar"/>
</antcall>
+ <antcall target="download-via-maven">
+ <param name="target" value="lib"/>
+ <param name="project-path" value="io/opencensus"/>
+ <param name="artifact-version" value="${opencensus.version}"/>
+ <param name="artifact-name" value="opencensus-api"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven">
+ <param name="target" value="lib"/>
+ <param name="project-path" value="io/opencensus"/>
+ <param name="artifact-version" value="${opencensus.version}"/>
+ <param name="artifact-name" value="opencensus-contrib-http-util"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven">
+ <param name="target" value="lib"/>
+ <param name="project-path" value="io/grpc"/>
+ <param name="artifact-version" value="${grpc.version}"/>
+ <param name="artifact-name" value="grpc-context"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
</target>
diff --git a/connectors/googledrive/build.xml b/connectors/googledrive/build.xml
index 976b8f63f..675dd910d 100644
--- a/connectors/googledrive/build.xml
+++ b/connectors/googledrive/build.xml
@@ -1,62 +1,64 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<project name="googledrive" default="all">
-
- <property environment="env"/>
- <condition property="mcf-dist" value="${env.MCFDISTPATH}">
- <isset property="env.MCFDISTPATH"/>
- </condition>
- <property name="abs-dist" location="../../dist"/>
- <condition property="mcf-dist" value="${abs-dist}">
- <not>
- <isset property="env.MCFDISTPATH"/>
- </not>
- </condition>
-
- <import file="${mcf-dist}/connector-build.xml"/>
-
- <path id="connector-classpath">
- <path refid="mcf-connector-build.connector-classpath"/>
- <fileset dir="../../lib">
- <include name="google-*.jar"/>
- <include name="jackson-core*.jar"/>
- <include name="jsr305*.jar"/>
- <include name="j2objc-annotations*.jar"/>
- </fileset>
- </path>
-
- <target name="lib" depends="mcf-connector-build.lib,precompile-check"
if="canBuild">
- <mkdir dir="dist/lib"/>
- <copy todir="dist/lib">
- <fileset dir="../../lib">
- <include name="google-*.jar"/>
- <include name="jackson-core*.jar"/>
- <include name="jsr305*.jar"/>
- <include name="j2objc-annotations*.jar"/>
- </fileset>
- </copy>
- </target>
-
- <target name="deliver-connector"
depends="mcf-connector-build.deliver-connector">
- <antcall target="general-add-repository-connector">
- <param name="connector-label" value="GoogleDrive"/>
- <param name="connector-class"
value="org.apache.manifoldcf.crawler.connectors.googledrive.GoogleDriveRepositoryConnector"/>
- </antcall>
- </target>
-
-</project>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project name="googledrive" default="all">
+
+ <property environment="env"/>
+ <condition property="mcf-dist" value="${env.MCFDISTPATH}">
+ <isset property="env.MCFDISTPATH"/>
+ </condition>
+ <property name="abs-dist" location="../../dist"/>
+ <condition property="mcf-dist" value="${abs-dist}">
+ <not>
+ <isset property="env.MCFDISTPATH"/>
+ </not>
+ </condition>
+
+ <import file="${mcf-dist}/connector-build.xml"/>
+
+ <path id="connector-classpath">
+ <path refid="mcf-connector-build.connector-classpath"/>
+ <fileset dir="../../lib">
+ <include name="google-*.jar"/>
+ <include name="jackson-core*.jar"/>
+ <include name="jsr305*.jar"/>
+ <include name="j2objc-annotations*.jar"/>
+ </fileset>
+ </path>
+
+ <target name="lib" depends="mcf-connector-build.lib,precompile-check"
if="canBuild">
+ <mkdir dir="dist/lib"/>
+ <copy todir="dist/lib">
+ <fileset dir="../../lib">
+ <include name="google-*.jar"/>
+ <include name="jackson-core*.jar"/>
+ <include name="jsr305*.jar"/>
+ <include name="j2objc-annotations*.jar"/>
+ <include name="grpc-context*.jar"/>
+ <include name="opencensus*.jar"/>
+ </fileset>
+ </copy>
+ </target>
+
+ <target name="deliver-connector"
depends="mcf-connector-build.deliver-connector">
+ <antcall target="general-add-repository-connector">
+ <param name="connector-label" value="GoogleDrive"/>
+ <param name="connector-class"
value="org.apache.manifoldcf.crawler.connectors.googledrive.GoogleDriveRepositoryConnector"/>
+ </antcall>
+ </target>
+
+</project>
\ No newline at end of file
diff --git
a/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java
b/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java
index c28bb55cc..cc999782d 100644
---
a/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java
+++
b/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java
@@ -1,3 +1,4 @@
+
/* $Id$ */
/**
@@ -54,10 +55,9 @@ import
org.apache.manifoldcf.crawler.interfaces.IProcessActivity;
import org.apache.manifoldcf.crawler.interfaces.ISeedingActivity;
import org.apache.manifoldcf.crawler.system.Logging;
-import com.google.api.client.repackaged.com.google.common.base.Objects;
+import com.google.common.base.Objects;
import com.google.api.client.util.DateTime;
import com.google.api.services.drive.model.File;
-import com.google.api.services.drive.model.ParentReference;
/**
*
@@ -981,33 +981,24 @@ public class GoogleDriveRepositoryConnector extends
BaseRepositoryConnector {
for (String documentIdentifier : documentIdentifiers) {
File googleFile = getObject(documentIdentifier);
-
- // StringBuilder log = new StringBuilder();
- // log.append("File Original Name: " + googleFile.getOriginalFilename());
- // log.append(System.getProperty("line.separator"));
- // log.append("File Title: " + googleFile.getTitle());
- // log.append(System.getProperty("line.separator"));
- // log.append("File Description: " + googleFile.getDescription());
- // log.append(System.getProperty("line.separator"));
- // log.append("File Extension: " + googleFile.getFileExtension());
- // log.append(System.getProperty("line.separator"));
- // log.append("File MimeType: " + googleFile.getMimeType());
- // log.append(System.getProperty("line.separator"));
- // log.append("File Version: " + googleFile.getVersion());
- // log.append(System.getProperty("line.separator"));
- //
- // System.out.println(log);
-
+
+ if (Logging.connectors.isDebugEnabled()) {
+ Logging.connectors.debug("GOOGLEDRIVE: Processing document identifier
'"
+ + documentIdentifier + "'");
+ Logging.connectors.debug("GOOGLEDRIVE: have this file:\t" +
googleFile.getName());
+ }
+
String versionString;
- if (googleFile == null || (googleFile.containsKey("explicitlyTrashed")
&& googleFile.getExplicitlyTrashed())) {
+ if (googleFile == null || (googleFile.containsKey("explicitlyTrashed")
&& googleFile.getExplicitlyTrashed()) ||
+
googleFile.getMimeType().equals("application/vnd.google-apps.shortcut")) {
//its deleted, move on
activities.deleteDocument(documentIdentifier);
continue;
}
if (!isDir(googleFile)) {
- String rev = googleFile.getModifiedDate().toStringRfc3339();
+ String rev = googleFile.getModifiedTime().toStringRfc3339();
if (StringUtils.isNotEmpty(rev)) {
StringBuilder sb = new StringBuilder();
@@ -1046,7 +1037,7 @@ public class GoogleDriveRepositoryConnector extends
BaseRepositoryConnector {
if (Logging.connectors.isDebugEnabled()) {
Logging.connectors.debug("GOOGLEDRIVE: Processing document
identifier '"
+ nodeId + "'");
- Logging.connectors.debug("GOOGLEDRIVE: have this file:\t" +
googleFile.getTitle());
+ Logging.connectors.debug("GOOGLEDRIVE: have this file:\t" +
googleFile.getName());
}
if
("application/vnd.google-apps.folder".equals(googleFile.getMimeType())) {
@@ -1110,16 +1101,16 @@ public class GoogleDriveRepositoryConnector extends
BaseRepositoryConnector {
}
// Get the file length
- Long fileLengthLong =
Objects.firstNonNull(googleFile.getFileSize(), 0L);
+ Long fileLengthLong = googleFile.getSize() != null ?
googleFile.getSize() : 0L;
if (fileLengthLong != null) {
// Now do standard stuff
long fileLength = fileLengthLong.longValue();
String mimeType = googleFile.getMimeType();
- DateTime createdDateObject = googleFile.getCreatedDate();
- DateTime modifiedDateObject = googleFile.getModifiedDate();
+ DateTime createdDateObject = googleFile.getCreatedTime();
+ DateTime modifiedDateObject = googleFile.getModifiedTime();
String extension = googleFile.getFileExtension();
- String title = cleanupFileFolderName(googleFile.getTitle());
+ String title = cleanupFileFolderName(googleFile.getName());
Date createdDate = (createdDateObject==null)?null:new
Date(createdDateObject.getValue());
Date modifiedDate = (modifiedDateObject==null)?null:new
Date(modifiedDateObject.getValue());
// We always direct to the PDF except for Spreadsheets
@@ -1353,14 +1344,14 @@ public class GoogleDriveRepositoryConnector extends
BaseRepositoryConnector {
try {
if (!isDir(googleFile)) {
if (googleFile.getParents() != null &&
!googleFile.getParents().isEmpty()) {
- ParentReference parentRef = googleFile.getParents().get(0);
+ String parentRef = googleFile.getParents().get(0);
File parent;
- parent = getObject(parentRef.getId());
+ parent = getObject(parentRef);
String path = getFilePath(parent);
String name;
- String title = cleanupFileFolderName(googleFile.getTitle());
+ String title = cleanupFileFolderName(googleFile.getName());
String extension = googleFile.getFileExtension();
@@ -1387,7 +1378,7 @@ public class GoogleDriveRepositoryConnector extends
BaseRepositoryConnector {
}
} else {
String path = getFilePath(googleFile);
- String name = cleanupFileFolderName(googleFile.getTitle());
+ String name = cleanupFileFolderName(googleFile.getName());
fullContentPath = path + SLASH + name;
}
} catch (ManifoldCFException e) {
@@ -1407,7 +1398,7 @@ public class GoogleDriveRepositoryConnector extends
BaseRepositoryConnector {
String folderPath = "";
String fullFilePath = null;
- List<ParentReference> parentReferencesList = file.getParents();
+ List<String> parentReferencesList = file.getParents();
List<String> folderList = new ArrayList<String>();
List<String> finalFolderList = getfoldersList(parentReferencesList,
folderList);
@@ -1417,21 +1408,22 @@ public class GoogleDriveRepositoryConnector extends
BaseRepositoryConnector {
folderPath += "/" + folder;
}
- fullFilePath = folderPath + "/" + cleanupFileFolderName(file.getTitle());
+ fullFilePath = folderPath + "/" + cleanupFileFolderName(file.getName());
return fullFilePath;
}
- private List<String> getfoldersList(List<ParentReference>
parentReferencesList, List<String> folderList)
+ private List<String> getfoldersList(List<String> parentReferencesList,
List<String> folderList)
throws IOException, ManifoldCFException, ServiceInterruption {
- for (int i = 0; i < parentReferencesList.size(); i++) {
- String id = parentReferencesList.get(i).getId();
+
+ for (int i = 0; parentReferencesList != null && i <
parentReferencesList.size(); i++) {
+ String id = parentReferencesList.get(i);
File file = getObject(id);
- folderList.add(cleanupFileFolderName(file.getTitle()));
+ folderList.add(cleanupFileFolderName(file.getName()));
- if (!(file.getParents().isEmpty())) {
- List<ParentReference> parentReferenceslist2 = file.getParents();
+ if (file.getParents() != null && !(file.getParents().isEmpty())) {
+ List<String> parentReferenceslist2 = file.getParents();
getfoldersList(parentReferenceslist2, folderList);
}
}
@@ -1491,7 +1483,7 @@ public class GoogleDriveRepositoryConnector extends
BaseRepositoryConnector {
*/
protected static String getUrl(File googleFile, String exportType) {
if (googleFile.containsKey("fileSize")) {
- return googleFile.getDownloadUrl();
+ return googleFile.getWebContentLink();
} else {
return (googleFile.getExportLinks() != null) ?
googleFile.getExportLinks().get(exportType) : null;
}
diff --git
a/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java
b/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java
index fd235c3fe..bbd2ae3a3 100644
---
a/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java
+++
b/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java
@@ -105,10 +105,13 @@ public class GoogleDriveSession {
request = drive.files().list().setQ(googleDriveQuery);
request.setIncludeTeamDriveItems(true);
request.setSupportsTeamDrives(true);
+ request.setSupportsAllDrives(true);
+
+ request.setFields("*");
do {
FileList files = request.execute();
- for (File f : files.getItems()) {
+ for (File f : files.getFiles()) {
idBuffer.add(f.getId());
}
request.setPageToken(files.getNextPageToken());
@@ -121,8 +124,11 @@ public class GoogleDriveSession {
public File getObject(String id) throws IOException {
Get get = drive.files().get(id);
get.setSupportsTeamDrives(true);
+ get.setSupportsAllDrives(true);
+ get.setFields("*");
File file = get.execute();
+
return file;
}
@@ -133,10 +139,13 @@ public class GoogleDriveSession {
Drive.Files.List request = drive.files().list().setQ("'" + nodeId + "' in
parents");
request.setIncludeTeamDriveItems(true);
request.setSupportsTeamDrives(true);
+ request.setSupportsAllDrives(true);
+
+ request.setFields("*");
do {
FileList files = request.execute();
- for (File f : files.getItems()) {
+ for (File f : files.getFiles()) {
idBuffer.add(f.getId());
}
request.setPageToken(files.getNextPageToken());
@@ -161,4 +170,4 @@ public class GoogleDriveSession {
}
}
-}
+}
\ No newline at end of file
diff --git a/connectors/googledrive/pom.xml b/connectors/googledrive/pom.xml
index 170b7f310..73a9139e5 100644
--- a/connectors/googledrive/pom.xml
+++ b/connectors/googledrive/pom.xml
@@ -40,8 +40,8 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <project.http.version>1.25.0</project.http.version>
- <project.oauth.version>1.25.0</project.oauth.version>
+ <project.http.version>1.42.1</project.http.version>
+ <project.oauth.version>1.34.1</project.oauth.version>
</properties>
<artifactId>mcf-googledrive-connector</artifactId>
@@ -380,7 +380,7 @@
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-drive</artifactId>
- <version>v2-rev326-1.25.0</version>
+ <version>v3-rev20220815-2.0.0</version>
</dependency>
<dependency>
<groupId>com.google.http-client</groupId>
@@ -398,4 +398,4 @@
<version>${project.oauth.version}</version>
</dependency>
</dependencies>
-</project>
+</project>
\ No newline at end of file
