diff --git a/Directory.Build.props b/Directory.Build.props index 29ed770..d708663 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -29,8 +29,8 @@ https://github.com/managedcode/Storage https://github.com/managedcode/Storage Managed Code - Storage - 10.1.12 - 10.1.12 + 10.1.13 + 10.1.13 diff --git a/ManagedCode.Storage.Core/Extensions/StorageTransferExtensions.cs b/ManagedCode.Storage.Core/Extensions/StorageTransferExtensions.cs new file mode 100644 index 0000000..be41658 --- /dev/null +++ b/ManagedCode.Storage.Core/Extensions/StorageTransferExtensions.cs @@ -0,0 +1,112 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using ManagedCode.Communication; +using ManagedCode.Storage.Core.Models; + +namespace ManagedCode.Storage.Core.Extensions; + +/// Transfers files between configured storage instances. +public static class StorageTransferExtensions +{ + /// Copies the source file to a vacant destination path and returns the destination metadata. + /// Callers must coordinate other writes and deletes to either path during the transfer. + /// The destination provider controls which content types and custom metadata it can persist. + public static Task> CopyToAsync(this IStorage source, string sourcePath, + IStorage destination, string destinationPath, CancellationToken cancellationToken = default) => + TransferAsync(source, sourcePath, destination, destinationPath, move: false, cancellationToken); + + /// Copies the source file and removes it after the destination write succeeds. + /// Callers must coordinate other writes and deletes to either path during the transfer. + /// If source deletion fails after the copy, the operation reports failure and keeps the destination file. + public static Task> MoveToAsync(this IStorage source, string sourcePath, + IStorage destination, string destinationPath, CancellationToken cancellationToken = default) => + TransferAsync(source, sourcePath, destination, destinationPath, move: true, cancellationToken); + + private static async Task> TransferAsync(IStorage source, string sourcePath, + IStorage destination, string destinationPath, bool move, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(source); + ArgumentNullException.ThrowIfNull(destination); + ArgumentException.ThrowIfNullOrWhiteSpace(sourcePath); + ArgumentException.ThrowIfNullOrWhiteSpace(destinationPath); + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var copied = await CopyFileAsync(source, sourcePath, destination, destinationPath, cancellationToken).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + if (!move || copied.IsFailed) + return copied; + + var deleted = await source.DeleteAsync(sourcePath, cancellationToken).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + if (deleted.IsFailed) + return Result.Fail(deleted.Problem); + + return deleted.Value + ? copied + : Result.Fail("The destination was written, but the source deletion was not confirmed."); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + cancellationToken.ThrowIfCancellationRequested(); + return Result.Fail(exception); + } + } + + private static async Task> CopyFileAsync(IStorage source, string sourcePath, + IStorage destination, string destinationPath, CancellationToken cancellationToken) + { + var metadata = await source.GetBlobMetadataAsync(sourcePath, cancellationToken).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + if (metadata.IsFailed) + return metadata; + + var exists = await destination.ExistsAsync(destinationPath, cancellationToken).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + if (exists.IsFailed) + return Result.Fail(exists.Problem); + if (exists.Value) + return Result.Fail("Destination already exists", destinationPath, HttpStatusCode.Conflict); + + var opened = await source.GetStreamAsync(sourcePath, cancellationToken).ConfigureAwait(false); + if (opened.IsFailed) + return Result.Fail(opened.Problem); + + var content = opened.Value; + await using (content.ConfigureAwait(false)) + { + cancellationToken.ThrowIfCancellationRequested(); + var info = metadata.Value; + var customMetadata = info.Metadata is null ? null : new Dictionary(info.Metadata, info.Metadata.Comparer); + var options = new UploadOptions(destinationPath, mimeType: info.MimeType, metadata: customMetadata); + + if (content.CanSeek) + return await UploadAsync(destination, content, options, info.Length, cancellationToken).ConfigureAwait(false); + + // Destination SDKs may need to determine the length or rewind for retries. + var staged = await LocalFile.FromStreamAsync(content, cancellationToken).ConfigureAwait(false); + await using (staged.ConfigureAwait(false)) + { + return await UploadAsync(destination, staged.FileStream, options, info.Length, cancellationToken).ConfigureAwait(false); + } + } + } + + private static async Task> UploadAsync(IStorage destination, Stream content, + UploadOptions options, ulong expectedLength, CancellationToken cancellationToken) + { + if ((ulong)content.Length != expectedLength) + return Result.Fail("Source length differs from its storage metadata."); + + var uploaded = await destination.UploadAsync(content, options, cancellationToken).ConfigureAwait(false); + if (uploaded.IsSuccess && uploaded.Value.Length != expectedLength) + return Result.Fail("Destination length differs from the source length."); + + return uploaded; + } +} diff --git a/README.md b/README.md index d79035f..9c5e531 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ Cross-provider blob storage toolkit for .NET and ASP.NET streaming scenarios. - [Packages](#packages) - [Cartograph artifacts](#cartograph-artifacts) - [Architecture](#architecture) +- [Copy and move between storages](#copy-and-move-between-storages) - [Virtual File System (VFS)](#virtual-file-system-vfs) - [Dependency Injection & Keyed Registrations](#dependency-injection--keyed-registrations) - [Orleans Grain Persistence](#orleans-grain-persistence) @@ -511,6 +512,19 @@ sequenceDiagram Controllers remain thin: consumers can inherit and override actions to add custom routing, authorization, or telemetry while leaving the streaming plumbing intact. +## Copy and move between storages + +Pass two configured `IStorage` instances and the relative paths in each storage. + +```csharp +using ManagedCode.Storage.Core.Extensions; + +var copy = await incomingStorage.CopyToAsync("uploads/report.pdf", archiveStorage, "2026/report.pdf", cancellationToken); +var move = await incomingStorage.MoveToAsync("uploads/raw.csv", archiveStorage, "2026/raw.csv", cancellationToken); +``` + +Both operations return `Result` for the destination. A move deletes the source only after the destination write succeeds. See the [transfer contract](docs/Features/storage-core.md#3-copy-and-move-between-storages) for coordination and recovery. + ## Virtual File System (VFS) Want a file/directory API on top of any configured `IStorage` (with optional metadata caching)? The ManagedCode.Storage.VirtualFileSystem package provides `IVirtualFileSystem`, which routes all operations through your registered storage provider. diff --git a/Tests/ManagedCode.Storage.Tests/Storages/Azure/AzureCrossStorageTransferTests.cs b/Tests/ManagedCode.Storage.Tests/Storages/Azure/AzureCrossStorageTransferTests.cs new file mode 100644 index 0000000..0038399 --- /dev/null +++ b/Tests/ManagedCode.Storage.Tests/Storages/Azure/AzureCrossStorageTransferTests.cs @@ -0,0 +1,69 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using ManagedCode.Storage.Azure; +using ManagedCode.Storage.Azure.Options; +using ManagedCode.Storage.Core.Extensions; +using ManagedCode.Storage.Core.Models; +using ManagedCode.Storage.FileSystem; +using ManagedCode.Storage.FileSystem.Options; +using ManagedCode.Storage.Tests.Common; +using Shouldly; +using Testcontainers.Azurite; +using Xunit; + +namespace ManagedCode.Storage.Tests.Storages.Azure; + +public sealed class AzureCrossStorageTransferTests : IAsyncLifetime +{ + private readonly AzuriteContainer _container = new AzuriteBuilder(ContainerImages.Azurite) + .WithCommand("--skipApiVersionCheck").Build(); + + public ValueTask InitializeAsync() => new(_container.StartAsync(TestContext.Current.CancellationToken)); + + public ValueTask DisposeAsync() => _container.DisposeAsync(); + + [Fact] + public async Task CopiesAzurePropertiesThenMovesTheFileToLocalStorage() + { + var token = TestContext.Current.CancellationToken; + using var source = CreateStorage(); + using var archive = CreateStorage(); + var folder = Directory.CreateTempSubdirectory("storage-transfer-"); + try + { + using var destination = new FileSystemStorage(new FileSystemStorageOptions { BaseFolder = folder.FullName }); + const string original = "incoming/дані.dat"; + const string copy = "copied/content.txt"; + const string moved = "moved/content.bin"; + const string contentType = "application/x-storage-test"; + var payload = "cross-storage payload"u8.ToArray(); + var options = new UploadOptions(original, mimeType: contentType, metadata: new() { ["author"] = "Марія" }); + (await source.UploadAsync(payload, options, token)).IsSuccess.ShouldBeTrue(); + + var copied = await source.CopyToAsync(original, archive, copy, token); + copied.IsSuccess.ShouldBeTrue(copied.Problem?.Detail); + var persisted = await archive.GetBlobMetadataAsync(copy, token); + persisted.IsSuccess.ShouldBeTrue(persisted.Problem?.Detail); + persisted.Value.MimeType.ShouldBe(contentType); + persisted.Value.Metadata.ShouldNotBeNull()["author"].ShouldBe("Марія"); + + var relocated = await archive.MoveToAsync(copy, destination, moved, token); + relocated.IsSuccess.ShouldBeTrue(relocated.Problem?.Detail); + (await File.ReadAllBytesAsync(Path.Combine(folder.FullName, moved), token)).ShouldBe(payload); + (await source.ExistsAsync(original, token)).Value.ShouldBeTrue(); + (await archive.ExistsAsync(copy, token)).Value.ShouldBeFalse(); + } + finally + { + folder.Delete(recursive: true); + } + } + + private AzureStorage CreateStorage() => new(new AzureStorageOptions + { + ConnectionString = _container.GetConnectionString(), + Container = $"transfer-{Guid.NewGuid():N}", + CreateContainerIfNotExists = true + }); +} diff --git a/Tests/ManagedCode.Storage.Tests/Storages/CrossStorageTransferTests.cs b/Tests/ManagedCode.Storage.Tests/Storages/CrossStorageTransferTests.cs new file mode 100644 index 0000000..3f914c8 --- /dev/null +++ b/Tests/ManagedCode.Storage.Tests/Storages/CrossStorageTransferTests.cs @@ -0,0 +1,176 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using ManagedCode.Storage.Core; +using ManagedCode.Storage.Core.Extensions; +using ManagedCode.Storage.Core.Models; +using ManagedCode.Storage.FileSystem; +using ManagedCode.Storage.FileSystem.Options; +using ManagedCode.Storage.Tests.Storages.Cartograph; +using Shouldly; +using Xunit; + +namespace ManagedCode.Storage.Tests.Storages; + +public sealed class CrossStorageTransferTests : IDisposable +{ + private const string SourcePath = "incoming/дані.txt"; + private const string DestinationPath = "archive/renamed.txt"; + private readonly DirectoryInfo _root = Directory.CreateTempSubdirectory("storage-transfer-"); + + [Theory] + [InlineData(false, "")] + [InlineData(false, "content to copy")] + [InlineData(true, "")] + [InlineData(true, "content to move")] + public async Task TransfersBetweenRootsAndOnlyMoveRemovesSource(bool move, string content) + { + var token = TestContext.Current.CancellationToken; + using var source = CreateStorage("source"); + using var destination = CreateStorage("destination"); + (await source.UploadAsync(content, new UploadOptions(SourcePath), token)).IsSuccess.ShouldBeTrue(); + + var result = move + ? await source.MoveToAsync(SourcePath, destination, DestinationPath, token) + : await source.CopyToAsync(SourcePath, destination, DestinationPath, token); + + result.IsSuccess.ShouldBeTrue(result.Problem?.Detail); + result.Value.FullName.ShouldBe(DestinationPath); + await AssertContentAsync(destination, DestinationPath, content, token); + (await source.ExistsAsync(SourcePath, token)).Value.ShouldBe(!move); + if (!move) + await AssertContentAsync(source, SourcePath, content, token); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExistingDestinationKeepsBothFiles(bool alias) + { + var token = TestContext.Current.CancellationToken; + using var source = CreateStorage("source"); + using var destination = CreateStorage(alias ? "source" : "destination"); + var target = alias ? SourcePath : DestinationPath; + (await source.UploadAsync("original", new UploadOptions(SourcePath), token)).IsSuccess.ShouldBeTrue(); + if (!alias) + (await destination.UploadAsync("existing", new UploadOptions(target), token)).IsSuccess.ShouldBeTrue(); + + var result = await source.MoveToAsync(SourcePath, destination, target, token); + + result.IsFailed.ShouldBeTrue(); + result.Problem.StatusCode.ShouldBe(409); + await AssertContentAsync(source, SourcePath, "original", token); + await AssertContentAsync(destination, target, alias ? "original" : "existing", token); + } + + [Fact] + public async Task MissingSourceDoesNotCreateDestination() + { + var token = TestContext.Current.CancellationToken; + using var source = CreateStorage("source"); + using var destination = CreateStorage("destination"); + + var result = await source.CopyToAsync(SourcePath, destination, DestinationPath, token); + + result.IsFailed.ShouldBeTrue(); + result.Problem.StatusCode.ShouldBe(404); + (await destination.ExistsAsync(DestinationPath, token)).Value.ShouldBeFalse(); + } + + [Fact] + public async Task RejectedUploadLeavesSourceReadable() + { + var token = TestContext.Current.CancellationToken; + using var source = CreateStorage("source"); + var folder = _root.CreateSubdirectory("destination"); + folder.CreateSubdirectory(DestinationPath); + using var destination = CreateStorage("destination"); + (await source.UploadAsync("original", new UploadOptions(SourcePath), token)).IsSuccess.ShouldBeTrue(); + + var result = await source.MoveToAsync(SourcePath, destination, DestinationPath, token); + + result.IsFailed.ShouldBeTrue(); + Directory.Exists(Path.Combine(folder.FullName, DestinationPath)).ShouldBeTrue(); + await AssertContentAsync(source, SourcePath, "original", token); + } + + [Fact] + public async Task RejectedDeleteLeavesTheCompletedCopyAndSource() + { + var token = TestContext.Current.CancellationToken; + using var artifact = new CartographArtifactFixture(); + using var source = artifact.CreateStorage(); + using var destination = CreateStorage("destination"); + + var result = await source.MoveToAsync("root.txt", destination, DestinationPath, token); + + result.IsFailed.ShouldBeTrue(); + const string expected = "Cartograph through Storage\n"; + await AssertContentAsync(source, "root.txt", expected, token); + await AssertContentAsync(destination, DestinationPath, expected, token); + } + + [Fact] + public async Task NonSeekableProviderCopiesMultiRecordFile() + { + var token = TestContext.Current.CancellationToken; + using var artifact = new CartographArtifactFixture(); + using var source = artifact.CreateStorage(); + using var destination = CreateStorage("destination"); + const string original = "reports/large.bin"; + const string target = "archive/large.bin"; + + var result = await source.CopyToAsync(original, destination, target, token); + + result.IsSuccess.ShouldBeTrue(result.Problem?.Detail); + (await File.ReadAllBytesAsync(Path.Combine(_root.FullName, "destination", target), token)) + .ShouldBe(artifact.Files[original]); + } + + [Fact] + public async Task FailedSequentialReadLeavesDestinationVacant() + { + var token = TestContext.Current.CancellationToken; + using var artifact = new CartographArtifactFixture(); + artifact.CorruptFirstPayload(); + using var source = artifact.CreateStorage(); + using var destination = CreateStorage("destination"); + + var result = await source.CopyToAsync("root.txt", destination, DestinationPath, token); + + result.IsFailed.ShouldBeTrue(); + (await destination.ExistsAsync(DestinationPath, token)).Value.ShouldBeFalse(); + } + + [Fact] + public async Task CancelledMoveLeavesSourceAndDoesNotCreateDestination() + { + var token = TestContext.Current.CancellationToken; + using var source = CreateStorage("source"); + using var destination = CreateStorage("destination"); + (await source.UploadAsync("original", new UploadOptions(SourcePath), token)).IsSuccess.ShouldBeTrue(); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + await Should.ThrowAsync(() => + source.MoveToAsync(SourcePath, destination, DestinationPath, cancellation.Token)); + + await AssertContentAsync(source, SourcePath, "original", token); + (await destination.ExistsAsync(DestinationPath, token)).Value.ShouldBeFalse(); + } + + private FileSystemStorage CreateStorage(string folder) => + new(new FileSystemStorageOptions { BaseFolder = _root.CreateSubdirectory(folder).FullName }); + + private static async Task AssertContentAsync(IStorage storage, string path, string expected, CancellationToken token) + { + var result = await storage.GetStreamAsync(path, token); + result.IsSuccess.ShouldBeTrue(result.Problem?.Detail); + await using var stream = result.Value; + using var reader = new StreamReader(stream, leaveOpen: true); + (await reader.ReadToEndAsync(token)).ShouldBe(expected); + } + + public void Dispose() => _root.Delete(recursive: true); +} diff --git a/docs/Architecture.md b/docs/Architecture.md index 3e0bde7..5b79134 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -13,6 +13,9 @@ Single source of truth: keep this doc navigational and coarse. Detailed behavior - **Entry points:** DI extension methods, `IStorage` and `IStorageFactory`, ASP.NET controllers and hubs, HTTP and SignalR clients, Orleans `IGrainStorage`, and the xUnit test suite. - **Dependencies:** providers implement the Core contracts; VFS and integrations consume `IStorage` or `IStorageFactory`; tests compose the real packages plus `ManagedCode.Storage.TestFakes/`. +Core composes two configured `IStorage` instances for +[cross-storage copy and move](Features/storage-core.md#3-copy-and-move-between-storages). + ## Atomic object capabilities `IStorage` retains its existing portable API. AzureStorage additionally implements diff --git a/docs/Features/storage-core.md b/docs/Features/storage-core.md index 9508da1..74c0b3b 100644 --- a/docs/Features/storage-core.md +++ b/docs/Features/storage-core.md @@ -37,6 +37,28 @@ flowchart LR - `MimeType` (normalized through `MimeHelper`) - per-provider metadata where supported +### 3) Copy and move between storages + +`CopyToAsync` and `MoveToAsync` in `ManagedCode.Storage.Core.Extensions` take a source +storage and path, a destination storage and path, and an optional cancellation token. +The storages can use different providers. Each successful call returns the destination's +`BlobMetadata`. + +```mermaid +flowchart LR + Source["Read source metadata"] --> Check["Check that destination is vacant"] + Check --> Upload["Upload content and verify length"] + Upload --> Done["Return destination metadata for copy"] + Upload --> Delete["Delete source for move"] + Delete --> DoneMove["Return destination metadata after deletion"] +``` + +- If the destination already contains a file at the given path, the operation returns a `409` conflict. +- Content type and custom metadata travel through the destination provider's upload options. What persists depends on the provider. +- Seekable sources stream directly to the destination. Sequential sources use a temporary local file that the transfer disposes, so providers can determine the length and rewind during upload. +- Coordinate access so other writers and deleters cannot modify either path during the operation. The existence check, upload, and delete are separate provider calls. +- A failed or cancelled write preserves the source. A provider can leave data at the destination if its upload fails after writing. If source deletion fails after a completed upload, the destination stays available. Before retrying a partial move, verify both paths and reconcile the existing destination. + ## Components Key files: @@ -48,6 +70,7 @@ Key files: - `ManagedCode.Storage.Core/Helpers/PathHelper.cs` — shared path manipulation utilities - `ManagedCode.Storage.Core/Providers/*` — provider factory abstractions and the default `StorageFactory` - `ManagedCode.Storage.Core/Extensions/ServiceCollectionExtensions.cs` — DI helpers +- [`StorageTransferExtensions`](https://github.com/managed-code-hub/Storage/blob/main/ManagedCode.Storage.Core/Extensions/StorageTransferExtensions.cs) — cross-storage file copy and move ## Current Behavior