|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one |
| 3 | + * or more contributor license agreements. See the NOTICE file |
| 4 | + * distributed with this work for additional information |
| 5 | + * regarding copyright ownership. The ASF licenses this file |
| 6 | + * to you under the Apache License, Version 2.0 (the |
| 7 | + * "License"); you may not use this file except in compliance |
| 8 | + * with the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, |
| 13 | + * software distributed under the License is distributed on an |
| 14 | + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | + * KIND, either express or implied. See the License for the |
| 16 | + * specific language governing permissions and limitations |
| 17 | + * under the License. |
| 18 | + */ |
| 19 | + |
| 20 | +#include "iceberg/arrow/arrow_file_io.h" |
| 21 | +#include "iceberg/result.h" |
| 22 | + |
| 23 | +#ifdef ICEBERG_S3_ENABLED |
| 24 | + |
| 25 | +# include <mutex> |
| 26 | + |
| 27 | +# include <arrow/filesystem/s3fs.h> |
| 28 | + |
| 29 | +# include "iceberg/arrow/arrow_fs_file_io_internal.h" |
| 30 | +# include "iceberg/arrow/arrow_status_internal.h" |
| 31 | +# include "iceberg/arrow/s3_properties.h" |
| 32 | +# include "iceberg/util/macros.h" |
| 33 | + |
| 34 | +namespace iceberg::arrow { |
| 35 | + |
| 36 | +namespace { |
| 37 | + |
| 38 | +Status EnsureS3Initialized() { |
| 39 | + static std::once_flag init_flag; |
| 40 | + static ::arrow::Status init_status; |
| 41 | + std::call_once(init_flag, [] { |
| 42 | + // TODO(smaheshwar): support options from ::arrow::fs::S3GlobalOptions when needed |
| 43 | + init_status = ::arrow::fs::EnsureS3Initialized(); |
| 44 | + }); |
| 45 | + if (!init_status.ok()) { |
| 46 | + return IOError("Arrow S3 initialization failed: {}", init_status.ToString()); |
| 47 | + } |
| 48 | + return {}; |
| 49 | +} |
| 50 | + |
| 51 | +std::string_view GetProperty( |
| 52 | + const std::unordered_map<std::string, std::string>& properties, |
| 53 | + std::string_view key) { |
| 54 | + if (auto it = properties.find(std::string(key)); it != properties.end()) { |
| 55 | + return it->second; |
| 56 | + } |
| 57 | + return {}; |
| 58 | +} |
| 59 | + |
| 60 | +Result<::arrow::fs::S3Options> ConfigureS3Options( |
| 61 | + const std::unordered_map<std::string, std::string>& properties) { |
| 62 | + ::arrow::fs::S3Options options; |
| 63 | + |
| 64 | + auto access_key = GetProperty(properties, S3Properties::kAccessKeyId); |
| 65 | + auto secret_key = GetProperty(properties, S3Properties::kSecretAccessKey); |
| 66 | + auto session_token = GetProperty(properties, S3Properties::kSessionToken); |
| 67 | + |
| 68 | + if (!access_key.empty() && !secret_key.empty()) { |
| 69 | + if (!session_token.empty()) { |
| 70 | + options.ConfigureAccessKey(std::string(access_key), std::string(secret_key), |
| 71 | + std::string(session_token)); |
| 72 | + } else { |
| 73 | + options.ConfigureAccessKey(std::string(access_key), std::string(secret_key)); |
| 74 | + } |
| 75 | + } else { |
| 76 | + options.ConfigureDefaultCredentials(); |
| 77 | + } |
| 78 | + |
| 79 | + if (auto region = GetProperty(properties, S3Properties::kRegion); !region.empty()) { |
| 80 | + options.region = std::string(region); |
| 81 | + } |
| 82 | + |
| 83 | + if (auto endpoint = GetProperty(properties, S3Properties::kEndpoint); |
| 84 | + !endpoint.empty()) { |
| 85 | + options.endpoint_override = std::string(endpoint); |
| 86 | + } |
| 87 | + |
| 88 | + if (GetProperty(properties, S3Properties::kPathStyleAccess) == "true") { |
| 89 | + options.force_virtual_addressing = false; |
| 90 | + } |
| 91 | + |
| 92 | + if (GetProperty(properties, S3Properties::kSslEnabled) == "false") { |
| 93 | + options.scheme = "http"; |
| 94 | + } |
| 95 | + |
| 96 | + return options; |
| 97 | +} |
| 98 | + |
| 99 | +} // namespace |
| 100 | + |
| 101 | +// TODO(smaheshwar): Expose a FinalizeS3() wrapper so users don't need arrow headers |
| 102 | +// directly. Arrow's S3 subsystem should be finalized at shutdown. |
| 103 | +Result<std::unique_ptr<FileIO>> MakeS3FileIO( |
| 104 | + const std::unordered_map<std::string, std::string>& properties) { |
| 105 | + ICEBERG_RETURN_UNEXPECTED(EnsureS3Initialized()); |
| 106 | + |
| 107 | + ICEBERG_ASSIGN_OR_RAISE(auto options, ConfigureS3Options(properties)); |
| 108 | + ICEBERG_ARROW_ASSIGN_OR_RETURN(auto fs, ::arrow::fs::S3FileSystem::Make(options)); |
| 109 | + |
| 110 | + return std::make_unique<ArrowFileSystemFileIO>( std::move(fs)); |
| 111 | +} |
| 112 | + |
| 113 | +} // namespace iceberg::arrow |
| 114 | + |
| 115 | +#else // !ICEBERG_S3_ENABLED |
| 116 | + |
| 117 | +namespace iceberg::arrow { |
| 118 | + |
| 119 | +Result<std::unique_ptr<FileIO>> MakeS3FileIO( |
| 120 | + const std::unordered_map<std::string, std::string>& /*properties*/) { |
| 121 | + return NotSupported("Arrow S3 support is not enabled"); |
| 122 | +} |
| 123 | + |
| 124 | +} // namespace iceberg::arrow |
| 125 | + |
| 126 | +#endif // ICEBERG_S3_ENABLED |
0 commit comments