nadment commented on code in PR #9533:
URL: https://github.com/apache/hop/pull/9533#discussion_r4605270864


##########
assemblies/plugins/pom.xml:
##########
@@ -1839,6 +1839,12 @@
                     <version>${project.version}</version>
                     <type>zip</type>
                 </dependency>
+                <dependency>
+                    <groupId>org.apache.hop</groupId>
+                    <artifactId>hop-tech-microsoft365</artifactId>

Review Comment:
   Fixed in 8c2c22ff9c: added the `hop-tech-microsoft365` entry to 
`optional-plugins.yaml` (category Technology, `installPath`/`modulePath: 
plugins/tech/microsoft365`) and the "(marketplace plugin 
`hop-tech-microsoft365`)" mention in the `vfs.adoc` table.



##########
plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/metadatatype/SharePointConnection.java:
##########
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.vfs.sharepoint.metadatatype;
+
+import com.azure.core.credential.TokenCredential;
+import com.azure.identity.ClientCertificateCredentialBuilder;
+import com.azure.identity.ClientSecretCredentialBuilder;
+import com.azure.identity.DefaultAzureCredentialBuilder;
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.vfs.IVfsBrowseLocation;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.HopMetadata;
+import org.apache.hop.metadata.api.HopMetadataBase;
+import org.apache.hop.metadata.api.HopMetadataCategory;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
+import org.apache.hop.metadata.api.IEnumHasCode;
+import org.apache.hop.metadata.api.IEnumHasCodeAndDescription;
+import org.apache.hop.metadata.api.IHopMetadata;
+import org.jspecify.annotations.NullMarked;
+import org.jspecify.annotations.Nullable;
+
+@Getter
+@Setter
+@NullMarked
+@GuiPlugin(classLoaderGroup = "vfs-graph")
+@HopMetadata(
+    key = "sharepoint",

Review Comment:
   Fixed in 8c2c22ff9c: the key is now `sharepoint-connection`, in line with 
the other connection types.



##########
plugins/tech/microsoft365/pom.xml:
##########
@@ -0,0 +1,263 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~       http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.apache.hop</groupId>
+        <artifactId>hop-plugins-tech</artifactId>
+        <version>2.21.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>hop-tech-microsoft365</artifactId>
+    <packaging>jar</packaging>
+    <name>Hop Plugins Technology Microsoft 365</name>
+
+    <properties>
+        <azure-eventhubs.version>3.3.0</azure-eventhubs.version>
+        <azure-graph.version>6.58.0</azure-graph.version>
+    </properties>
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>io.netty</groupId>
+                <artifactId>netty-bom</artifactId>
+                <version>${netty.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+            <dependency>
+                <groupId>com.azure</groupId>
+                <artifactId>azure-sdk-bom</artifactId>
+                <version>${azure-sdk-bom.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.hop</groupId>
+                <artifactId>hop-libs</artifactId>
+                <version>${project.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.microsoft.graph</groupId>
+            <artifactId>microsoft-graph</artifactId>
+            <version>${azure-graph.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.microsoft.graph</groupId>
+            <artifactId>microsoft-graph-core</artifactId>
+            <version>3.6.5</version>
+        </dependency>
+        <dependency>
+            <groupId>com.microsoft.kiota</groupId>
+            <artifactId>microsoft-kiota-abstractions</artifactId>
+            <version>1.9.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.microsoft.kiota</groupId>
+            <artifactId>microsoft-kiota-authentication-azure</artifactId>
+            <version>1.9.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.microsoft.kiota</groupId>
+            <artifactId>microsoft-kiota-http-okHttp</artifactId>
+            <version>1.9.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.microsoft.kiota</groupId>
+            <artifactId>microsoft-kiota-serialization-form</artifactId>
+            <version>1.9.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.microsoft.kiota</groupId>
+            <artifactId>microsoft-kiota-serialization-json</artifactId>
+            <version>1.9.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.microsoft.kiota</groupId>
+            <artifactId>microsoft-kiota-serialization-multipart</artifactId>
+            <version>1.9.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.microsoft.kiota</groupId>
+            <artifactId>microsoft-kiota-serialization-text</artifactId>
+            <version>1.9.0</version>
+        </dependency>
+        <dependency>
+            <groupId>io.github.std-uritemplate</groupId>
+            <artifactId>std-uritemplate</artifactId>
+            <version>2.0.0</version>
+        </dependency>
+        <!-- Shared with the other plugins through lib/core -->
+        <dependency>
+            <groupId>com.azure</groupId>
+            <artifactId>azure-core</artifactId>

Review Comment:
   Fixed in 8c2c22ff9c: `com.fasterxml.jackson.datatype:*` is now excluded from 
`azure-core`, as in the Azure plugin.



##########
plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileSystem.java:
##########
@@ -0,0 +1,503 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hop.vfs.sharepoint;
+
+import com.microsoft.graph.core.models.UploadResult;
+import com.microsoft.graph.core.tasks.LargeFileUploadTask;
+import com.microsoft.graph.drives.item.items.item.DriveItemItemRequestBuilder;
+import 
com.microsoft.graph.drives.item.items.item.children.ChildrenRequestBuilder;
+import 
com.microsoft.graph.drives.item.items.item.createuploadsession.CreateUploadSessionPostRequestBody;
+import com.microsoft.graph.models.BaseCollectionPaginationCountResponse;
+import com.microsoft.graph.models.DriveCollectionResponse;
+import com.microsoft.graph.models.DriveItem;
+import com.microsoft.graph.models.DriveItemCollectionResponse;
+import com.microsoft.graph.models.DriveItemUploadableProperties;
+import com.microsoft.graph.models.Folder;
+import com.microsoft.graph.models.ItemReference;
+import com.microsoft.graph.models.Site;
+import com.microsoft.graph.models.SiteCollectionResponse;
+import com.microsoft.graph.models.UploadSession;
+import com.microsoft.graph.models.odataerrors.ODataError;
+import com.microsoft.graph.serviceclient.GraphServiceClient;
+import com.microsoft.graph.sites.item.drives.DrivesRequestBuilder;
+import com.microsoft.kiota.ApiException;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import org.apache.commons.vfs2.Capability;
+import org.apache.commons.vfs2.FileName;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemOptions;
+import org.apache.commons.vfs2.provider.AbstractFileName;
+import org.apache.commons.vfs2.provider.AbstractFileSystem;
+import org.apache.hop.core.logging.LogChannel;
+import org.jspecify.annotations.NullMarked;
+import org.jspecify.annotations.Nullable;
+
+/**
+ * One named Microsoft 365 connection. The sites are the folders of the root, 
the document libraries
+ * (drives) of a site are the folders of that site.
+ */
+@NullMarked
+public class SharePointFileSystem extends AbstractFileSystem {
+
+  public static final List<Capability> CAPABILITIES =
+      List.of(
+          Capability.CREATE,
+          Capability.DELETE,
+          Capability.RENAME,
+          Capability.GET_TYPE,
+          Capability.GET_LAST_MODIFIED,
+          Capability.LIST_CHILDREN,
+          Capability.READ_CONTENT,
+          Capability.URI,
+          Capability.WRITE_CONTENT);
+
+  /** Above this size the content is sent in slices through an upload session. 
*/
+  static final long SIMPLE_UPLOAD_LIMIT = 4L * 1024 * 1024;
+
+  /** The id that addresses the root folder of a document library, and the 
root site of a tenant. */
+  static final String ROOT_ITEM_ID = "root";
+
+  /** A site created by a Microsoft Teams team may live under /teams. */
+  private static final List<String> MANAGED_PATHS = List.of("sites", "teams");
+
+  /** The only properties of a file or folder that are used, the others are 
not requested. */
+  private static final String[] ITEM_PROPERTIES = {
+    "id", "name", "size", "lastModifiedDateTime", "file", "folder", "package"
+  };
+
+  private static final String CONFLICT_BEHAVIOR = 
"@microsoft.graph.conflictBehavior";
+  private static final int HTTP_NOT_FOUND = 404;
+  private static final int HTTP_CONFLICT = 409;
+
+  private final GraphServiceClient client;
+
+  /** Site name to site id, whatever the case of the name, null until the 
sites are listed. */
+  private volatile @Nullable Map<String, String> siteIds;
+
+  /** The SharePoint host of the tenant, null until it is needed. */
+  private volatile @Nullable String hostname;
+
+  /** False once Microsoft Graph refused to list the sites. */
+  private volatile boolean sitesListable = true;
+
+  /** Site id to (drive name to drive id, whatever the case of the name). */
+  private final Map<String, Map<String, String>> driveIds = new 
ConcurrentHashMap<>();
+
+  public SharePointFileSystem(
+      FileName rootName,
+      GraphServiceClient client,
+      @Nullable String hostname,
+      @Nullable FileSystemOptions fileSystemOptions) {
+    super(rootName, null, fileSystemOptions);
+    this.client = client;
+    this.hostname = hostname == null || hostname.isBlank() ? null : 
hostname.trim();
+  }
+
+  @Override
+  protected void addCapabilities(Collection<Capability> capabilities) {
+    capabilities.addAll(CAPABILITIES);
+  }
+
+  @Override
+  protected FileObject createFile(AbstractFileName name) {
+    return new SharePointFileObject(name, this);
+  }
+
+  /**
+   * Lists the sites of the tenant and refreshes the cache.
+   *
+   * @return the site ids by site name
+   */
+  Map<String, String> listSites() throws FileSystemException {
+    Map<String, String> sites = newNameMap();
+    try {
+      addSites(sites, client.sites().get());
+    } catch (ApiException e) {
+      throw new FileSystemException(
+          "Microsoft Graph refused to list the SharePoint sites: " + 
describe(e), e);
+    }
+    if (sites.isEmpty()) {
+      // Depending on the tenant the plain listing is empty and only the 
search returns the sites
+      try {
+        String url = client.getRequestAdapter().getBaseUrl() + 
"/sites?search=*";
+        addSites(sites, client.sites().withUrl(url).get());
+      } catch (ApiException e) {
+        LogChannel.GENERAL.logBasic(
+            "Microsoft Graph refused to search the SharePoint sites: " + 
describe(e));
+      }
+    }
+    if (sites.isEmpty()) {
+      LogChannel.GENERAL.logBasic(
+          "Microsoft Graph returned no SharePoint site. Listing the sites 
needs the application"
+              + " permission Sites.Read.All with admin consent. With 
Sites.Selected the sites"
+              + " can not be listed, use the full path of a site the 
application was granted.");
+    }
+    siteIds = sites;
+    driveIds.clear();
+    return sites;
+  }
+
+  /**
+   * @return the HTTP status, the error code and the message of Microsoft Graph
+   */
+  static String describe(ApiException e) {
+    StringBuilder text = new StringBuilder("HTTP 
").append(e.getResponseStatusCode());
+    if (e instanceof ODataError error && error.getError() != null) {
+      text.append(' ').append(error.getError().getCode());
+      text.append(": ").append(error.getError().getMessage());
+    } else if (e.getMessage() != null) {
+      text.append(": ").append(e.getMessage());
+    }
+    return text.toString();
+  }
+
+  private void addSites(Map<String, String> sites, @Nullable 
SiteCollectionResponse firstPage) {
+    forEachPaged(
+        firstPage,
+        SiteCollectionResponse::getValue,
+        nextLink -> client.sites().withUrl(nextLink).get(),
+        site -> addSite(sites, site));
+  }
+
+  /** Two sites can have the same name, only the first one is kept. */
+  private static void addSite(Map<String, String> sites, Site site) {
+    String name = site.getName() != null ? site.getName() : 
site.getDisplayName();
+    if (name == null || site.getId() == null) {
+      return;
+    }
+    String namesake = sites.putIfAbsent(name, site.getId());
+    if (namesake != null && !namesake.equals(site.getId())) {
+      LogChannel.GENERAL.logBasic(
+          "The SharePoint site "
+              + site.getWebUrl()
+              + " is ignored: another site is already named '"
+              + name
+              + "'");
+    }
+  }
+
+  /**
+   * Reads a site by its address, {@code /sites/name} of the SharePoint host 
of the tenant. This
+   * also works when the application can not list the sites (Sites.Selected).
+   *
+   * @return the id of the site or null if it does not exist or can not be read
+   */
+  private @Nullable String findSiteIdByPath(String siteName) {
+    String host;
+    try {
+      host = hostname();
+    } catch (ApiException e) {
+      LogChannel.GENERAL.logBasic(
+          "The SharePoint host of the tenant can not be read, set it on the 
connection: "
+              + describe(e));
+      return null;
+    }
+    if (host == null) {
+      return null;
+    }
+    String encodedSiteName = SharePointFileName.encodeSegment(siteName);
+    for (String managedPath : MANAGED_PATHS) {
+      // The address of the site relative to the host, such as 
contoso.sharepoint.com:/sites/Finance
+      String address = host + ":/" + managedPath + "/";
+      String url = client.getRequestAdapter().getBaseUrl() + "/sites/" + 
address + encodedSiteName;
+      try {
+        Site site = client.sites().bySiteId(ROOT_ITEM_ID).withUrl(url).get();
+        if (site != null && site.getId() != null) {
+          return site.getId();
+        }
+      } catch (ApiException e) {
+        LogChannel.GENERAL.logBasic(
+            "The SharePoint site "
+                + host
+                + "/"
+                + managedPath
+                + "/"
+                + siteName
+                + " can not be read: "
+                + describe(e));
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Lists the sites, an application that is only granted some sites 
(Sites.Selected) can not list
+   * them and reaches its sites by their address.
+   */
+  private Map<String, String> listSitesIfAllowed() {
+    try {
+      return listSites();
+    } catch (FileSystemException e) {
+      LogChannel.GENERAL.logDetailed(e.getMessage());
+      Map<String, String> sites = newNameMap();
+      sitesListable = false;
+      siteIds = sites;
+      return sites;
+    }
+  }
+
+  /**
+   * @return the SharePoint host of the tenant, such as 
contoso.sharepoint.com, or null if the root
+   *     site can not be read
+   */
+  private @Nullable String hostname() {
+    if (hostname == null) {
+      Site root = client.sites().bySiteId(ROOT_ITEM_ID).get();
+      if (root != null && root.getSiteCollection() != null) {
+        hostname = root.getSiteCollection().getHostname();
+      }
+      if (hostname == null && root != null && root.getWebUrl() != null) {
+        hostname = URI.create(root.getWebUrl()).getHost();
+      }
+    }
+    return hostname;
+  }
+
+  /**
+   * @return the id of the site or null if there is no site with this name
+   */
+  @Nullable String findSiteId(String siteName) throws FileSystemException {

Review Comment:
   Fixed in 8c2c22ff9c: `findSiteId` now tries `findSiteIdByPath` first 
(`/sites/<site>`, then `/teams/<site>`) and caches the result. The listing is 
kept for browsing `name:///` and as a fallback when the address gives nothing, 
and that fallback runs at most once per file system, so a site that doesn't 
exist no longer lists them all again on every lookup.
   
   Two side effects: a 404 on one of the two managed paths is now logged at 
detailed level, since it is expected for a site under `/teams`; and when two 
sites have the same name, the one under `/sites` wins. The docs are updated 
accordingly. Tested against a real tenant (read, write and folder listing).



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to