Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Directory.Build.props
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@
<RepositoryUrl>https://github.com/managedcode/Storage</RepositoryUrl>
<PackageProjectUrl>https://github.com/managedcode/Storage</PackageProjectUrl>
<Product>Managed Code - Storage</Product>
<Version>10.1.12</Version>
<PackageVersion>10.1.12</PackageVersion>
<Version>10.1.13</Version>
<PackageVersion>10.1.13</PackageVersion>

</PropertyGroup>

Expand Down
112 changes: 112 additions & 0 deletions ManagedCode.Storage.Core/Extensions/StorageTransferExtensions.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using ManagedCode.Communication;
using ManagedCode.Storage.Core.Models;

namespace ManagedCode.Storage.Core.Extensions;

/// <summary>Transfers files between configured storage instances.</summary>
public static class StorageTransferExtensions
{
/// <summary>Copies the source file to a vacant destination path and returns the destination metadata.</summary>
/// <remarks>Callers must coordinate other writes and deletes to either path during the transfer.
/// The destination provider controls which content types and custom metadata it can persist.</remarks>
public static Task<Result<BlobMetadata>> CopyToAsync(this IStorage source, string sourcePath,
IStorage destination, string destinationPath, CancellationToken cancellationToken = default) =>
TransferAsync(source, sourcePath, destination, destinationPath, move: false, cancellationToken);

/// <summary>Copies the source file and removes it after the destination write succeeds.</summary>
/// <remarks>Callers must coordinate other writes and deletes to either path during the transfer.
/// If source deletion fails after the copy, the operation reports failure and keeps the destination file.</remarks>
public static Task<Result<BlobMetadata>> MoveToAsync(this IStorage source, string sourcePath,
IStorage destination, string destinationPath, CancellationToken cancellationToken = default) =>
TransferAsync(source, sourcePath, destination, destinationPath, move: true, cancellationToken);

private static async Task<Result<BlobMetadata>> TransferAsync(IStorage source, string sourcePath,
IStorage destination, string destinationPath, bool move, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(source);
ArgumentNullException.ThrowIfNull(destination);
ArgumentException.ThrowIfNullOrWhiteSpace(sourcePath);
ArgumentException.ThrowIfNullOrWhiteSpace(destinationPath);
cancellationToken.ThrowIfCancellationRequested();

try
{
var copied = await CopyFileAsync(source, sourcePath, destination, destinationPath, cancellationToken).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
if (!move || copied.IsFailed)
return copied;

var deleted = await source.DeleteAsync(sourcePath, cancellationToken).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
if (deleted.IsFailed)
return Result<BlobMetadata>.Fail(deleted.Problem);

return deleted.Value
? copied
: Result<BlobMetadata>.Fail("The destination was written, but the source deletion was not confirmed.");
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
cancellationToken.ThrowIfCancellationRequested();
return Result<BlobMetadata>.Fail(exception);
}
}

private static async Task<Result<BlobMetadata>> CopyFileAsync(IStorage source, string sourcePath,
IStorage destination, string destinationPath, CancellationToken cancellationToken)
{
var metadata = await source.GetBlobMetadataAsync(sourcePath, cancellationToken).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
if (metadata.IsFailed)
return metadata;

var exists = await destination.ExistsAsync(destinationPath, cancellationToken).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
if (exists.IsFailed)
return Result<BlobMetadata>.Fail(exists.Problem);
if (exists.Value)
return Result<BlobMetadata>.Fail("Destination already exists", destinationPath, HttpStatusCode.Conflict);

var opened = await source.GetStreamAsync(sourcePath, cancellationToken).ConfigureAwait(false);
if (opened.IsFailed)
return Result<BlobMetadata>.Fail(opened.Problem);

var content = opened.Value;
await using (content.ConfigureAwait(false))
{
cancellationToken.ThrowIfCancellationRequested();
var info = metadata.Value;
var customMetadata = info.Metadata is null ? null : new Dictionary<string, string>(info.Metadata, info.Metadata.Comparer);
var options = new UploadOptions(destinationPath, mimeType: info.MimeType, metadata: customMetadata);

if (content.CanSeek)
return await UploadAsync(destination, content, options, info.Length, cancellationToken).ConfigureAwait(false);

// Destination SDKs may need to determine the length or rewind for retries.
var staged = await LocalFile.FromStreamAsync(content, cancellationToken).ConfigureAwait(false);
await using (staged.ConfigureAwait(false))
{
return await UploadAsync(destination, staged.FileStream, options, info.Length, cancellationToken).ConfigureAwait(false);
}
}
}

private static async Task<Result<BlobMetadata>> UploadAsync(IStorage destination, Stream content,
UploadOptions options, ulong expectedLength, CancellationToken cancellationToken)
{
if ((ulong)content.Length != expectedLength)
return Result<BlobMetadata>.Fail("Source length differs from its storage metadata.");

var uploaded = await destination.UploadAsync(content, options, cancellationToken).ConfigureAwait(false);
if (uploaded.IsSuccess && uploaded.Value.Length != expectedLength)
return Result<BlobMetadata>.Fail("Destination length differs from the source length.");

return uploaded;
}
}
14 changes: 14 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ Cross-provider blob storage toolkit for .NET and ASP.NET streaming scenarios.
- [Packages](#packages)
- [Cartograph artifacts](#cartograph-artifacts)
- [Architecture](#architecture)
- [Copy and move between storages](#copy-and-move-between-storages)
- [Virtual File System (VFS)](#virtual-file-system-vfs)
- [Dependency Injection & Keyed Registrations](#dependency-injection--keyed-registrations)
- [Orleans Grain Persistence](#orleans-grain-persistence)
Expand Down Expand Up @@ -511,6 +512,19 @@ sequenceDiagram

Controllers remain thin: consumers can inherit and override actions to add custom routing, authorization, or telemetry while leaving the streaming plumbing intact.

## Copy and move between storages

Pass two configured `IStorage` instances and the relative paths in each storage.

```csharp
using ManagedCode.Storage.Core.Extensions;

var copy = await incomingStorage.CopyToAsync("uploads/report.pdf", archiveStorage, "2026/report.pdf", cancellationToken);
var move = await incomingStorage.MoveToAsync("uploads/raw.csv", archiveStorage, "2026/raw.csv", cancellationToken);
```

Both operations return `Result<BlobMetadata>` for the destination. A move deletes the source only after the destination write succeeds. See the [transfer contract](docs/Features/storage-core.md#3-copy-and-move-between-storages) for coordination and recovery.

## Virtual File System (VFS)

Want a file/directory API on top of any configured `IStorage` (with optional metadata caching)? The <code>ManagedCode.Storage.VirtualFileSystem</code> package provides `IVirtualFileSystem`, which routes all operations through your registered storage provider.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
using System;
using System.IO;
using System.Threading.Tasks;
using ManagedCode.Storage.Azure;
using ManagedCode.Storage.Azure.Options;
using ManagedCode.Storage.Core.Extensions;
using ManagedCode.Storage.Core.Models;
using ManagedCode.Storage.FileSystem;
using ManagedCode.Storage.FileSystem.Options;
using ManagedCode.Storage.Tests.Common;
using Shouldly;
using Testcontainers.Azurite;
using Xunit;

namespace ManagedCode.Storage.Tests.Storages.Azure;

public sealed class AzureCrossStorageTransferTests : IAsyncLifetime
{
private readonly AzuriteContainer _container = new AzuriteBuilder(ContainerImages.Azurite)
.WithCommand("--skipApiVersionCheck").Build();

public ValueTask InitializeAsync() => new(_container.StartAsync(TestContext.Current.CancellationToken));

public ValueTask DisposeAsync() => _container.DisposeAsync();

[Fact]
public async Task CopiesAzurePropertiesThenMovesTheFileToLocalStorage()
{
var token = TestContext.Current.CancellationToken;
using var source = CreateStorage();
using var archive = CreateStorage();
var folder = Directory.CreateTempSubdirectory("storage-transfer-");
try
{
using var destination = new FileSystemStorage(new FileSystemStorageOptions { BaseFolder = folder.FullName });
const string original = "incoming/дані.dat";
const string copy = "copied/content.txt";
const string moved = "moved/content.bin";
const string contentType = "application/x-storage-test";
var payload = "cross-storage payload"u8.ToArray();
var options = new UploadOptions(original, mimeType: contentType, metadata: new() { ["author"] = "Марія" });
(await source.UploadAsync(payload, options, token)).IsSuccess.ShouldBeTrue();

var copied = await source.CopyToAsync(original, archive, copy, token);
copied.IsSuccess.ShouldBeTrue(copied.Problem?.Detail);
var persisted = await archive.GetBlobMetadataAsync(copy, token);
persisted.IsSuccess.ShouldBeTrue(persisted.Problem?.Detail);
persisted.Value.MimeType.ShouldBe(contentType);
persisted.Value.Metadata.ShouldNotBeNull()["author"].ShouldBe("Марія");

var relocated = await archive.MoveToAsync(copy, destination, moved, token);
relocated.IsSuccess.ShouldBeTrue(relocated.Problem?.Detail);
(await File.ReadAllBytesAsync(Path.Combine(folder.FullName, moved), token)).ShouldBe(payload);
(await source.ExistsAsync(original, token)).Value.ShouldBeTrue();
(await archive.ExistsAsync(copy, token)).Value.ShouldBeFalse();
}
finally
{
folder.Delete(recursive: true);
}
}

private AzureStorage CreateStorage() => new(new AzureStorageOptions
{
ConnectionString = _container.GetConnectionString(),
Container = $"transfer-{Guid.NewGuid():N}",
CreateContainerIfNotExists = true
});
}
176 changes: 176 additions & 0 deletions Tests/ManagedCode.Storage.Tests/Storages/CrossStorageTransferTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using ManagedCode.Storage.Core;
using ManagedCode.Storage.Core.Extensions;
using ManagedCode.Storage.Core.Models;
using ManagedCode.Storage.FileSystem;
using ManagedCode.Storage.FileSystem.Options;
using ManagedCode.Storage.Tests.Storages.Cartograph;
using Shouldly;
using Xunit;

namespace ManagedCode.Storage.Tests.Storages;

public sealed class CrossStorageTransferTests : IDisposable
{
private const string SourcePath = "incoming/дані.txt";
private const string DestinationPath = "archive/renamed.txt";
private readonly DirectoryInfo _root = Directory.CreateTempSubdirectory("storage-transfer-");

[Theory]
[InlineData(false, "")]
[InlineData(false, "content to copy")]
[InlineData(true, "")]
[InlineData(true, "content to move")]
public async Task TransfersBetweenRootsAndOnlyMoveRemovesSource(bool move, string content)
{
var token = TestContext.Current.CancellationToken;
using var source = CreateStorage("source");
using var destination = CreateStorage("destination");
(await source.UploadAsync(content, new UploadOptions(SourcePath), token)).IsSuccess.ShouldBeTrue();

var result = move
? await source.MoveToAsync(SourcePath, destination, DestinationPath, token)
: await source.CopyToAsync(SourcePath, destination, DestinationPath, token);

result.IsSuccess.ShouldBeTrue(result.Problem?.Detail);
result.Value.FullName.ShouldBe(DestinationPath);
await AssertContentAsync(destination, DestinationPath, content, token);
(await source.ExistsAsync(SourcePath, token)).Value.ShouldBe(!move);
if (!move)
await AssertContentAsync(source, SourcePath, content, token);
}

[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task ExistingDestinationKeepsBothFiles(bool alias)
{
var token = TestContext.Current.CancellationToken;
using var source = CreateStorage("source");
using var destination = CreateStorage(alias ? "source" : "destination");
var target = alias ? SourcePath : DestinationPath;
(await source.UploadAsync("original", new UploadOptions(SourcePath), token)).IsSuccess.ShouldBeTrue();
if (!alias)
(await destination.UploadAsync("existing", new UploadOptions(target), token)).IsSuccess.ShouldBeTrue();

var result = await source.MoveToAsync(SourcePath, destination, target, token);

result.IsFailed.ShouldBeTrue();
result.Problem.StatusCode.ShouldBe(409);
await AssertContentAsync(source, SourcePath, "original", token);
await AssertContentAsync(destination, target, alias ? "original" : "existing", token);
}

[Fact]
public async Task MissingSourceDoesNotCreateDestination()
{
var token = TestContext.Current.CancellationToken;
using var source = CreateStorage("source");
using var destination = CreateStorage("destination");

var result = await source.CopyToAsync(SourcePath, destination, DestinationPath, token);

result.IsFailed.ShouldBeTrue();
result.Problem.StatusCode.ShouldBe(404);
(await destination.ExistsAsync(DestinationPath, token)).Value.ShouldBeFalse();
}

[Fact]
public async Task RejectedUploadLeavesSourceReadable()
{
var token = TestContext.Current.CancellationToken;
using var source = CreateStorage("source");
var folder = _root.CreateSubdirectory("destination");
folder.CreateSubdirectory(DestinationPath);
using var destination = CreateStorage("destination");
(await source.UploadAsync("original", new UploadOptions(SourcePath), token)).IsSuccess.ShouldBeTrue();

var result = await source.MoveToAsync(SourcePath, destination, DestinationPath, token);

result.IsFailed.ShouldBeTrue();
Directory.Exists(Path.Combine(folder.FullName, DestinationPath)).ShouldBeTrue();
await AssertContentAsync(source, SourcePath, "original", token);
}

[Fact]
public async Task RejectedDeleteLeavesTheCompletedCopyAndSource()
{
var token = TestContext.Current.CancellationToken;
using var artifact = new CartographArtifactFixture();
using var source = artifact.CreateStorage();
using var destination = CreateStorage("destination");

var result = await source.MoveToAsync("root.txt", destination, DestinationPath, token);

result.IsFailed.ShouldBeTrue();
const string expected = "Cartograph through Storage\n";
await AssertContentAsync(source, "root.txt", expected, token);
await AssertContentAsync(destination, DestinationPath, expected, token);
}

[Fact]
public async Task NonSeekableProviderCopiesMultiRecordFile()
{
var token = TestContext.Current.CancellationToken;
using var artifact = new CartographArtifactFixture();
using var source = artifact.CreateStorage();
using var destination = CreateStorage("destination");
const string original = "reports/large.bin";
const string target = "archive/large.bin";

var result = await source.CopyToAsync(original, destination, target, token);

result.IsSuccess.ShouldBeTrue(result.Problem?.Detail);
(await File.ReadAllBytesAsync(Path.Combine(_root.FullName, "destination", target), token))
.ShouldBe(artifact.Files[original]);
}

[Fact]
public async Task FailedSequentialReadLeavesDestinationVacant()
{
var token = TestContext.Current.CancellationToken;
using var artifact = new CartographArtifactFixture();
artifact.CorruptFirstPayload();
using var source = artifact.CreateStorage();
using var destination = CreateStorage("destination");

var result = await source.CopyToAsync("root.txt", destination, DestinationPath, token);

result.IsFailed.ShouldBeTrue();
(await destination.ExistsAsync(DestinationPath, token)).Value.ShouldBeFalse();
}

[Fact]
public async Task CancelledMoveLeavesSourceAndDoesNotCreateDestination()
{
var token = TestContext.Current.CancellationToken;
using var source = CreateStorage("source");
using var destination = CreateStorage("destination");
(await source.UploadAsync("original", new UploadOptions(SourcePath), token)).IsSuccess.ShouldBeTrue();
using var cancellation = new CancellationTokenSource();
cancellation.Cancel();

await Should.ThrowAsync<OperationCanceledException>(() =>
source.MoveToAsync(SourcePath, destination, DestinationPath, cancellation.Token));

await AssertContentAsync(source, SourcePath, "original", token);
(await destination.ExistsAsync(DestinationPath, token)).Value.ShouldBeFalse();
}

private FileSystemStorage CreateStorage(string folder) =>
new(new FileSystemStorageOptions { BaseFolder = _root.CreateSubdirectory(folder).FullName });

private static async Task AssertContentAsync(IStorage storage, string path, string expected, CancellationToken token)
{
var result = await storage.GetStreamAsync(path, token);
result.IsSuccess.ShouldBeTrue(result.Problem?.Detail);
await using var stream = result.Value;
using var reader = new StreamReader(stream, leaveOpen: true);
(await reader.ReadToEndAsync(token)).ShouldBe(expected);
}

public void Dispose() => _root.Delete(recursive: true);
}
Loading
Loading