srilman commented on code in PR #35701: URL: https://github.com/apache/arrow/pull/35701#discussion_r1248938339
########## cpp/src/arrow/filesystem/azurefs.h: ########## @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <memory> +#include <string> +#include <vector> + +#include "arrow/filesystem/filesystem.h" +#include "arrow/util/macros.h" +#include "arrow/util/uri.h" + +namespace Azure { +namespace Core { +namespace Credentials { + +class TokenCredential; + +} // namespace Credentials +} // namespace Core +namespace Storage { + +class StorageSharedKeyCredential; + +} // namespace Storage +} // namespace Azure + +namespace arrow { +namespace fs { + +enum class AzureCredentialsKind : int8_t { + /// Anonymous access (no credentials used), public + Anonymous, + /// Use explicitly-provided access key pair + StorageCredentials, + /// Use ServicePrincipleCredentials + ServicePrincipleCredentials, + /// Use Sas Token to authenticate + Sas, + /// Use Connection String + ConnectionString +}; + +/// Options for the AzureFileSystem implementation. +struct ARROW_EXPORT AzureOptions { + std::string scheme; + std::string account_dfs_url; + std::string account_blob_url; + bool is_azurite = false; + AzureCredentialsKind credentials_kind = AzureCredentialsKind::Anonymous; + + std::string sas_token; + std::string connection_string; + std::shared_ptr<Azure::Storage::StorageSharedKeyCredential> + storage_credentials_provider; + std::shared_ptr<Azure::Core::Credentials::TokenCredential> + service_principle_credentials_provider; + + AzureOptions(); + + bool Equals(const AzureOptions& other) const; +}; + +/// \brief Azure-backed FileSystem implementation for ABFS and ADLS. +/// +/// ABFS (Azure Blob Storage - https://azure.microsoft.com/en-us/products/storage/blobs/) +/// object-based cloud storage system. +/// +/// ADLS (Azure Data Lake Storage - https://azure.microsoft.com/en-us/products/storage/data-lake-storage/) +/// is a scalable data storage system designed for big-data applications. +/// ADLS provides filesystem semantics, file-level security, and Hadoop +/// compatibility. Gen1 exists as a separate object that will retired +/// on Feb 29, 2024. New ADLS accounts will use Gen2 instead, which is +/// implemented on top of ABFS. +/// +/// TODO: GH-18014 Complete the internal implementation +/// and review the documentation +class ARROW_EXPORT AzureFileSystem : public FileSystem { + public: + ~AzureFileSystem() override = default; + + std::string type_name() const override { return "abfs"; } + + /// Return the original Azure options when constructing the filesystem + AzureOptions options() const; Review Comment: Good point. `AzureFileSystem::Impl::options` already returns a const-ref anyways. ########## cpp/src/arrow/filesystem/azurefs.h: ########## @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <memory> +#include <string> +#include <vector> + +#include "arrow/filesystem/filesystem.h" +#include "arrow/util/macros.h" +#include "arrow/util/uri.h" + +namespace Azure { +namespace Core { +namespace Credentials { + +class TokenCredential; + +} // namespace Credentials +} // namespace Core +namespace Storage { + +class StorageSharedKeyCredential; + +} // namespace Storage +} // namespace Azure + +namespace arrow { +namespace fs { + +enum class AzureCredentialsKind : int8_t { + /// Anonymous access (no credentials used), public + Anonymous, + /// Use explicitly-provided access key pair + StorageCredentials, + /// Use ServicePrincipleCredentials + ServicePrincipleCredentials, + /// Use Sas Token to authenticate + Sas, + /// Use Connection String + ConnectionString +}; + +/// Options for the AzureFileSystem implementation. +struct ARROW_EXPORT AzureOptions { + std::string scheme; + std::string account_dfs_url; + std::string account_blob_url; + bool is_azurite = false; + AzureCredentialsKind credentials_kind = AzureCredentialsKind::Anonymous; + + std::string sas_token; + std::string connection_string; + std::shared_ptr<Azure::Storage::StorageSharedKeyCredential> + storage_credentials_provider; + std::shared_ptr<Azure::Core::Credentials::TokenCredential> + service_principle_credentials_provider; + + AzureOptions(); + + bool Equals(const AzureOptions& other) const; +}; + +/// \brief Azure-backed FileSystem implementation for ABFS and ADLS. +/// +/// ABFS (Azure Blob Storage - https://azure.microsoft.com/en-us/products/storage/blobs/) +/// object-based cloud storage system. +/// +/// ADLS (Azure Data Lake Storage - https://azure.microsoft.com/en-us/products/storage/data-lake-storage/) +/// is a scalable data storage system designed for big-data applications. +/// ADLS provides filesystem semantics, file-level security, and Hadoop +/// compatibility. Gen1 exists as a separate object that will retired +/// on Feb 29, 2024. New ADLS accounts will use Gen2 instead, which is +/// implemented on top of ABFS. +/// +/// TODO: GH-18014 Complete the internal implementation +/// and review the documentation +class ARROW_EXPORT AzureFileSystem : public FileSystem { + public: + ~AzureFileSystem() override = default; + + std::string type_name() const override { return "abfs"; } + + /// Return the original Azure options when constructing the filesystem + AzureOptions options() const; Review Comment: Changed -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
