Skip to content

Commit 529e567

Browse files
rameelrameel
authored
Merge pull request #63 from rameel/pkg-upgrade
Upgrade packages
2 parents 622e791 + 8adf021 commit 529e567

13 files changed

Lines changed: 131 additions & 78 deletions

File tree

‎.github/workflows/tests.yml‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,18 @@ jobs:
3636
azure:
3737
name: "Test: AzureFileSystem"
3838
runs-on: ubuntu-latest
39-
services:
40-
azurite:
41-
image: mcr.microsoft.com/azure-storage/azurite
42-
ports:
43-
- 10000:10000
44-
- 10001:10001
45-
- 10002:10002
4639
steps:
40+
- name: Start Azurite
41+
run: |
42+
docker run -d \
43+
-p 10000:10000 \
44+
-p 10001:10001 \
45+
-p 10002:10002 \
46+
mcr.microsoft.com/azure-storage/azurite \
47+
azurite --skipApiVersionCheck \
48+
--blobHost 0.0.0.0 \
49+
--queueHost 0.0.0.0 \
50+
--tableHost 0.0.0.0
4751
- name: Setup .NET
4852
uses: actions/setup-dotnet@v4
4953
with:
@@ -58,7 +62,7 @@ jobs:
5862
- name: Build
5963
run: dotnet build -c Debug
6064
- name: Test
61-
run: dotnet test --no-build --filter TestCategory=Cloud:Azure
65+
run: dotnet test --no-build --filter TestCategory=Cloud:Azure -maxcpucount:1
6266

6367
s3:
6468
name: "Test: S3FileSystem"

‎Directory.Packages.props‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@
44
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
55
</PropertyGroup>
66
<ItemGroup>
7-
<PackageVersion Include="AWSSDK.S3" Version="3.7.402.11" />
8-
<PackageVersion Include="Azure.Storage.Blobs" Version="12.21.2" />
9-
<PackageVersion Include="Azure.Storage.Blobs.Batch" Version="12.18.1" />
10-
<PackageVersion Include="Google.Cloud.Storage.V1" Version="4.10.0" />
11-
<PackageVersion Include="Microsoft.Extensions.FileProviders.Abstractions" Version="6.0.0" />
12-
<PackageVersion Include="Microsoft.Extensions.FileProviders.Physical" Version="6.0.0" />
7+
<PackageVersion Include="AWSSDK.S3" Version="4.0.18.6" />
8+
<PackageVersion Include="Azure.Storage.Blobs" Version="12.27.0" />
9+
<PackageVersion Include="Azure.Storage.Blobs.Batch" Version="12.24.0" />
10+
<PackageVersion Include="Google.Cloud.Storage.V1" Version="4.14.0" />
11+
<PackageVersion Include="Microsoft.Extensions.FileProviders.Abstractions" Version="6.0.1" />
12+
<PackageVersion Include="Microsoft.Extensions.FileProviders.Physical" Version="6.0.1" />
1313
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.13.0" />
1414
<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
1515
<PackageVersion Include="MinVer" Version="6.0.0" />
@@ -18,4 +18,4 @@
1818
<PackageVersion Include="Ramstack.Globbing" Version="2.3.3" />
1919
<PackageVersion Include="System.Linq.Async" Version="6.0.1" />
2020
</ItemGroup>
21-
</Project>
21+
</Project>

‎Ramstack.FileSystem.slnx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
<File Path=".gitignore" />
55
<File Path="Directory.Build.props" />
66
<File Path="Directory.Packages.props" />
7+
<File Path="docker-compose.yml" />
78
<File Path="LICENSE" />
89
<File Path="README.md" />
910
</Folder>

‎docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ services:
66
- "10000:10000"
77
- "10001:10001"
88
- "10002:10002"
9-
command: azurite --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0
9+
command: azurite --skipApiVersionCheck --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0
1010

1111
rustfs:
1212
image: docker.io/rustfs/rustfs:latest

‎src/Ramstack.FileSystem.Amazon/S3Directory.cs‎

Lines changed: 64 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ protected override async ValueTask DeleteCoreAsync(CancellationToken cancellatio
4848
BucketName = _fs.BucketName
4949
};
5050

51+
dr.Objects ??= [];
52+
5153
do
5254
{
5355
// The maximum number of objects returned is MaxKeys, which is 1000,
@@ -59,8 +61,9 @@ protected override async ValueTask DeleteCoreAsync(CancellationToken cancellatio
5961
.ListObjectsV2Async(lr, cancellationToken)
6062
.ConfigureAwait(false);
6163

62-
foreach (var obj in response.S3Objects)
63-
dr.Objects.Add(new KeyVersion { Key = obj.Key });
64+
if (response.S3Objects is not null)
65+
foreach (var obj in response.S3Objects)
66+
dr.Objects.Add(new KeyVersion { Key = obj.Key });
6467

6568
if (dr.Objects.Count != 0)
6669
await _fs.AmazonClient
@@ -89,11 +92,13 @@ protected override async IAsyncEnumerable<VirtualNode> GetFileNodesCoreAsync([En
8992
.ListObjectsV2Async(request, cancellationToken)
9093
.ConfigureAwait(false);
9194

92-
foreach (var prefix in response.CommonPrefixes)
93-
yield return new S3Directory(_fs, VirtualPath.Normalize(prefix));
95+
if (response.CommonPrefixes is not null)
96+
foreach (var prefix in response.CommonPrefixes)
97+
yield return new S3Directory(_fs, VirtualPath.Normalize(prefix));
9498

95-
foreach (var obj in response.S3Objects)
96-
yield return CreateVirtualFile(obj);
99+
if (response.S3Objects is not null)
100+
foreach (var obj in response.S3Objects)
101+
yield return CreateVirtualFile(obj);
97102

98103
request.ContinuationToken = response.NextContinuationToken;
99104
}
@@ -116,8 +121,9 @@ protected override async IAsyncEnumerable<VirtualFile> GetFilesCoreAsync([Enumer
116121
.ListObjectsV2Async(request, cancellationToken)
117122
.ConfigureAwait(false);
118123

119-
foreach (var obj in response.S3Objects)
120-
yield return CreateVirtualFile(obj);
124+
if (response.S3Objects is not null)
125+
foreach (var obj in response.S3Objects)
126+
yield return CreateVirtualFile(obj);
121127

122128
request.ContinuationToken = response.NextContinuationToken;
123129
}
@@ -140,8 +146,9 @@ protected override async IAsyncEnumerable<VirtualDirectory> GetDirectoriesCoreAs
140146
.ListObjectsV2Async(request, cancellationToken)
141147
.ConfigureAwait(false);
142148

143-
foreach (var prefix in response.CommonPrefixes)
144-
yield return new S3Directory(_fs, VirtualPath.Normalize(prefix));
149+
if (response.CommonPrefixes is not null)
150+
foreach (var prefix in response.CommonPrefixes)
151+
yield return new S3Directory(_fs, VirtualPath.Normalize(prefix));
145152

146153
request.ContinuationToken = response.NextContinuationToken;
147154
}
@@ -179,28 +186,31 @@ protected override async IAsyncEnumerable<VirtualNode> GetFileNodesCoreAsync(str
179186
.ListObjectsV2Async(request, cancellationToken)
180187
.ConfigureAwait(false);
181188

182-
foreach (var obj in response.S3Objects)
189+
if (response.S3Objects is not null)
183190
{
184-
var path = VirtualPath.Normalize(obj.Key);
185-
var directoryPath = VirtualPath.GetDirectoryName(path);
186-
187-
while (directoryPath.Length != 0 && directories.Add(directoryPath))
191+
foreach (var obj in response.S3Objects)
188192
{
189-
//
190-
// Directories are yielded in reverse order (deepest first).
191-
//
192-
// Note: We could use a Stack<string> to control the order,
193-
// but since order isn't guaranteed anyway and to avoid
194-
// unnecessary memory allocation, we process them directly.
195-
//
196-
if (IsMatched(directoryPath.AsSpan(FullName.Length), patterns, excludes))
197-
yield return new S3Directory(_fs, directoryPath);
198-
199-
directoryPath = VirtualPath.GetDirectoryName(directoryPath);
193+
var path = VirtualPath.Normalize(obj.Key);
194+
var directoryPath = VirtualPath.GetDirectoryName(path);
195+
196+
while (directoryPath.Length != 0 && directories.Add(directoryPath))
197+
{
198+
//
199+
// Directories are yielded in reverse order (deepest first).
200+
//
201+
// Note: We could use a Stack<string> to control the order,
202+
// but since order isn't guaranteed anyway and to avoid
203+
// unnecessary memory allocation, we process them directly.
204+
//
205+
if (IsMatched(directoryPath.AsSpan(FullName.Length), patterns, excludes))
206+
yield return new S3Directory(_fs, directoryPath);
207+
208+
directoryPath = VirtualPath.GetDirectoryName(directoryPath);
209+
}
210+
211+
if (IsMatched(obj.Key.AsSpan(request.Prefix.Length), patterns, excludes))
212+
yield return CreateVirtualFile(obj, path);
200213
}
201-
202-
if (IsMatched(obj.Key.AsSpan(request.Prefix.Length), patterns, excludes))
203-
yield return CreateVirtualFile(obj, path);
204214
}
205215

206216
request.ContinuationToken = response.NextContinuationToken;
@@ -234,9 +244,10 @@ protected override async IAsyncEnumerable<VirtualFile> GetFilesCoreAsync(string[
234244
.ListObjectsV2Async(request, cancellationToken)
235245
.ConfigureAwait(false);
236246

237-
foreach (var obj in response.S3Objects)
238-
if (IsMatched(obj.Key.AsSpan(request.Prefix.Length), patterns, excludes))
239-
yield return CreateVirtualFile(obj);
247+
if (response.S3Objects is not null)
248+
foreach (var obj in response.S3Objects)
249+
if (IsMatched(obj.Key.AsSpan(request.Prefix.Length), patterns, excludes))
250+
yield return CreateVirtualFile(obj);
240251

241252
request.ContinuationToken = response.NextContinuationToken;
242253
}
@@ -274,24 +285,27 @@ protected override async IAsyncEnumerable<VirtualDirectory> GetDirectoriesCoreAs
274285
.ListObjectsV2Async(request, cancellationToken)
275286
.ConfigureAwait(false);
276287

277-
foreach (var obj in response.S3Objects)
288+
if (response.S3Objects is not null)
278289
{
279-
var directoryPath = VirtualPath.GetDirectoryName(
280-
VirtualPath.Normalize(obj.Key));
281-
282-
while (directoryPath.Length != 0 && directories.Add(directoryPath))
290+
foreach (var obj in response.S3Objects)
283291
{
284-
//
285-
// Directories are yielded in reverse order (deepest first).
286-
//
287-
// Note: We could use a Stack<string> to control the order,
288-
// but since order isn't guaranteed anyway and to avoid
289-
// unnecessary memory allocation, we process them directly.
290-
//
291-
if (IsMatched(directoryPath.AsSpan(FullName.Length), patterns, excludes))
292-
yield return new S3Directory(_fs, directoryPath);
293-
294-
directoryPath = VirtualPath.GetDirectoryName(directoryPath);
292+
var directoryPath = VirtualPath.GetDirectoryName(
293+
VirtualPath.Normalize(obj.Key));
294+
295+
while (directoryPath.Length != 0 && directories.Add(directoryPath))
296+
{
297+
//
298+
// Directories are yielded in reverse order (deepest first).
299+
//
300+
// Note: We could use a Stack<string> to control the order,
301+
// but since order isn't guaranteed anyway and to avoid
302+
// unnecessary memory allocation, we process them directly.
303+
//
304+
if (IsMatched(directoryPath.AsSpan(FullName.Length), patterns, excludes))
305+
yield return new S3Directory(_fs, directoryPath);
306+
307+
directoryPath = VirtualPath.GetDirectoryName(directoryPath);
308+
}
295309
}
296310
}
297311

@@ -314,8 +328,8 @@ private S3File CreateVirtualFile(S3Object obj, string? normalizedName = null)
314328
.CreateFileProperties(
315329
creationTime: default,
316330
lastAccessTime: default,
317-
lastWriteTime: obj.LastModified,
318-
length: obj.Size);
331+
lastWriteTime: obj.LastModified.GetValueOrDefault(),
332+
length: obj.Size ?? 0);
319333

320334
var path = normalizedName ?? VirtualPath.Normalize(obj.Key);
321335
return new S3File(_fs, path, properties);

‎src/Ramstack.FileSystem.Amazon/S3File.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public S3File(AmazonS3FileSystem fileSystem, string path, VirtualNodeProperties?
4747
return VirtualNodeProperties.CreateFileProperties(
4848
creationTime: default,
4949
lastAccessTime: default,
50-
lastWriteTime: metadata.LastModified,
50+
lastWriteTime: metadata.LastModified.GetValueOrDefault(),
5151
length: metadata.ContentLength);
5252
}
5353
catch (AmazonS3Exception e) when (e.StatusCode == HttpStatusCode.NotFound)

‎src/Ramstack.FileSystem.Azure/AzureDirectory.cs‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ protected override async ValueTask DeleteCoreAsync(CancellationToken cancellatio
4747
var collection = _fs.AzureClient
4848
.GetBlobsAsync(
4949
prefix: GetPrefix(FullName),
50+
traits: BlobTraits.None,
51+
states: BlobStates.None,
5052
cancellationToken: cancellationToken);
5153

5254
var client = _fs.AzureClient.GetBlobBatchClient();
@@ -105,6 +107,8 @@ protected override async IAsyncEnumerable<VirtualNode> GetFileNodesCoreAsync([En
105107
.GetBlobsByHierarchyAsync(
106108
delimiter: "/",
107109
prefix: GetPrefix(FullName),
110+
traits: BlobTraits.None,
111+
states: BlobStates.None,
108112
cancellationToken: cancellationToken);
109113

110114
await foreach (var page in collection.AsPages().WithCancellation(cancellationToken).ConfigureAwait(false))
@@ -121,6 +125,8 @@ protected override async IAsyncEnumerable<VirtualFile> GetFilesCoreAsync([Enumer
121125
.GetBlobsByHierarchyAsync(
122126
delimiter: "/",
123127
prefix: GetPrefix(FullName),
128+
traits: BlobTraits.None,
129+
states: BlobStates.None,
124130
cancellationToken: cancellationToken);
125131

126132
await foreach (var page in collection.AsPages().WithCancellation(cancellationToken).ConfigureAwait(false))
@@ -136,6 +142,8 @@ protected override async IAsyncEnumerable<VirtualDirectory> GetDirectoriesCoreAs
136142
.GetBlobsByHierarchyAsync(
137143
delimiter: "/",
138144
prefix: GetPrefix(FullName),
145+
traits: BlobTraits.None,
146+
states: BlobStates.None,
139147
cancellationToken: cancellationToken);
140148

141149
await foreach (var page in collection.AsPages().WithCancellation(cancellationToken).ConfigureAwait(false))
@@ -162,7 +170,11 @@ protected override async IAsyncEnumerable<VirtualNode> GetFileNodesCoreAsync(str
162170
var directories = new HashSet<string> { FullName };
163171

164172
await foreach (var page in _fs.AzureClient
165-
.GetBlobsAsync(prefix: prefix, cancellationToken: cancellationToken)
173+
.GetBlobsAsync(
174+
prefix: prefix,
175+
traits: BlobTraits.None,
176+
states: BlobStates.None,
177+
cancellationToken: cancellationToken)
166178
.AsPages()
167179
.WithCancellation(cancellationToken)
168180
.ConfigureAwait(false))
@@ -210,7 +222,11 @@ protected override async IAsyncEnumerable<VirtualFile> GetFilesCoreAsync(string[
210222
var prefix = GetPrefix(FullName);
211223

212224
await foreach (var page in _fs.AzureClient
213-
.GetBlobsAsync(prefix: prefix, cancellationToken: cancellationToken)
225+
.GetBlobsAsync(
226+
prefix: prefix,
227+
traits: BlobTraits.None,
228+
states: BlobStates.None,
229+
cancellationToken: cancellationToken)
214230
.AsPages()
215231
.WithCancellation(cancellationToken)
216232
.ConfigureAwait(false))
@@ -239,7 +255,11 @@ protected override async IAsyncEnumerable<VirtualDirectory> GetDirectoriesCoreAs
239255
var directories = new HashSet<string> { FullName };
240256

241257
await foreach (var page in _fs.AzureClient
242-
.GetBlobsAsync(prefix: prefix, cancellationToken: cancellationToken)
258+
.GetBlobsAsync(
259+
prefix: prefix,
260+
traits: BlobTraits.None,
261+
states: BlobStates.None,
262+
cancellationToken: cancellationToken)
243263
.AsPages()
244264
.WithCancellation(cancellationToken)
245265
.ConfigureAwait(false))

‎src/Ramstack.FileSystem.Azure/Ramstack.FileSystem.Azure.csproj‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
4-
<TargetFramework>net6.0</TargetFramework>
4+
<TargetFrameworks>net6.0;net8.0</TargetFrameworks>
55
<Description>Provides an implementation of Ramstack.FileSystem based on Azure Blob Storage.</Description>
66
<ImplicitUsings>enable</ImplicitUsings>
77
<Nullable>enable</Nullable>
@@ -40,9 +40,17 @@
4040
</Compile>
4141
</ItemGroup>
4242

43-
<ItemGroup>
43+
<ItemGroup Condition="'$(TargetFramework)' == 'net6.0'">
44+
<PackageReference Include="Azure.Storage.Blobs" VersionOverride="12.21.2" />
45+
<PackageReference Include="Azure.Storage.Blobs.Batch" VersionOverride="12.18.1" />
46+
</ItemGroup>
47+
48+
<ItemGroup Condition="'$(TargetFramework)' == 'net8.0'">
4449
<PackageReference Include="Azure.Storage.Blobs" />
4550
<PackageReference Include="Azure.Storage.Blobs.Batch" />
51+
</ItemGroup>
52+
53+
<ItemGroup>
4654
<PackageReference Include="Microsoft.SourceLink.GitHub">
4755
<PrivateAssets>all</PrivateAssets>
4856
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

‎tests/Ramstack.FileSystem.Azure.Tests/Ramstack.FileSystem.Azure.Tests.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
4-
<TargetFramework>net6.0</TargetFramework>
4+
<TargetFrameworks>net6.0;net8.0</TargetFrameworks>
55
<ImplicitUsings>enable</ImplicitUsings>
66
<Nullable>enable</Nullable>
77
<LangVersion>preview</LangVersion>

0 commit comments

Comments
 (0)