From 23af590cb24175f8df509fa9e3d57e854e2e15d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 7 Apr 2024 21:19:13 +0200 Subject: [PATCH 1/5] Making it possible to use local S3 emulators --- .../AwsStorageOptionsExtension.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/OrchardCore.Modules/OrchardCore.Media.AmazonS3/AwsStorageOptionsExtension.cs b/src/OrchardCore.Modules/OrchardCore.Media.AmazonS3/AwsStorageOptionsExtension.cs index 4c45b3d30e2..40b1a8b9dca 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media.AmazonS3/AwsStorageOptionsExtension.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media.AmazonS3/AwsStorageOptionsExtension.cs @@ -2,6 +2,7 @@ using System.ComponentModel.DataAnnotations; using Amazon.Extensions.NETCore.Setup; using Amazon.Runtime; +using Amazon.S3; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using OrchardCore.Environment.Shell.Configuration; @@ -20,7 +21,7 @@ public static IEnumerable Validate(this AwsStorageOptions opti if (options.AwsOptions is not null) { - if (options.AwsOptions.Region is null) + if (options.AwsOptions.Region is null && options.AwsOptions.DefaultClientConfig.ServiceURL is null) { yield return new ValidationResult(Constants.ValidationMessages.RegionEndpointIsEmpty); } @@ -43,8 +44,9 @@ public static AwsStorageOptions BindConfiguration(this AwsStorageOptions options try { - // Binding AWS Options. - options.AwsOptions = shellConfiguration.GetAWSOptions("OrchardCore_Media_AmazonS3"); + // Binding AWS Options. Using the AmazonS3Config type parameter is necessary to be able to configure + // S3-specific properties like ForcePathStyle via the configuration provider. + options.AwsOptions = shellConfiguration.GetAWSOptions("OrchardCore_Media_AmazonS3"); // In case Credentials sections was specified, trying to add BasicAWSCredential to AWSOptions // since by design GetAWSOptions skips Credential section while parsing config. From aa1330f42aca261523a013564d61e141b76db867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 7 Apr 2024 21:19:32 +0200 Subject: [PATCH 2/5] Fixing S3 file uploads --- .../Controllers/AdminController.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Controllers/AdminController.cs b/src/OrchardCore.Modules/OrchardCore.Media/Controllers/AdminController.cs index 4085af7b85d..2344f85c752 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Controllers/AdminController.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Controllers/AdminController.cs @@ -215,7 +215,19 @@ public async Task Upload(string path, string extensions) var mediaFile = await _mediaFileStore.GetFileInfoAsync(mediaFilePath); - stream.Position = 0; + // The .NET AWS SDK, and only that form the built-in ones (but others maybe too), disposes + // the stream. There's no better way to check for that than handling the exception. An + // alternative would be to re-read the file for every other storage provider as well but + // that would be wasteful. + try + { + stream.Position = 0; + } + catch (ObjectDisposedException) + { + stream = null; + } + await PreCacheRemoteMedia(mediaFile, stream); result.Add(CreateFileResult(mediaFile)); From d0f41ab64ad645edb3b08ea06116ebb9465415a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 7 Apr 2024 21:22:30 +0200 Subject: [PATCH 3/5] Documenting using local S3 emulators --- .../modules/Media.AmazonS3/README.md | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/docs/reference/modules/Media.AmazonS3/README.md b/src/docs/reference/modules/Media.AmazonS3/README.md index 540e106ec8a..f2d8e003638 100644 --- a/src/docs/reference/modules/Media.AmazonS3/README.md +++ b/src/docs/reference/modules/Media.AmazonS3/README.md @@ -154,6 +154,51 @@ The `BucketName` property and the `BasePath` property are the only templatable p } ``` +## Configuring a Local Emulator + +During development, instead of using an online S3 resource, you can use a local storage emulator too. This is especially important for development teams, since you don't want to step on each others' feet by using a shared storage. + +For emulators, you'll need to configure a `ServiceURL`. Instead of the default [virtual host addressing of buckets](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html), you'll also need to enable path-style addressing (i.e. instead of `http://mybucket.localhost`, the local bucket will be accessible under `http://localhost/mybucket`). + +```json +{ + "OrchardCore": { + "OrchardCore_Media_AmazonS3": { + // Note how we use ServiceURL, just with emulators. There should be no Region specified. + "ServiceURL": "http://localhost:9444/", + "Profile": "default", + "ProfilesLocation": "", + // Providing some credentials is required but emulators don't actually use them. + "Credentials": { + "SecretKey": "dummy", + "AccessKey": "dummy" + }, + // BasePath and BucketName can be templated too, as usual. + "BasePath": "/media", + "CreateBucket": true, + "RemoveBucket": true, + "BucketName": "media", + // This is required for all emulators. + "ForcePathStyle": true + } + } +} +``` + +The following tools are known to be working with the above settings. Be sure to explore further configuration of these tools, the commands below are just provided for your convenience as a general recommendation. + +- [LocalS3](https://github.com/Robothy/local-s3) with Docker: + +``` +docker run -d -e MODE=IN_MEMORY -p 9444:80 luofuxiang/local-s3 +``` + +- [S3Mock](https://github.com/adobe/S3Mock) with Docker: + +``` +docker run -p 9444:9090 -t adobe/s3mock +``` + ## Media Cache The Media Cache feature will automatically be enabled when Amazon Media Storage is enabled. From 9aadde280f755e0d554254793906bbd595267560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 7 Apr 2024 21:31:42 +0200 Subject: [PATCH 4/5] Typo --- .../OrchardCore.Media/Controllers/AdminController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Controllers/AdminController.cs b/src/OrchardCore.Modules/OrchardCore.Media/Controllers/AdminController.cs index 2344f85c752..710af8c1616 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Controllers/AdminController.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Controllers/AdminController.cs @@ -215,7 +215,7 @@ public async Task Upload(string path, string extensions) var mediaFile = await _mediaFileStore.GetFileInfoAsync(mediaFilePath); - // The .NET AWS SDK, and only that form the built-in ones (but others maybe too), disposes + // The .NET AWS SDK, and only that from the built-in ones (but others maybe too), disposes // the stream. There's no better way to check for that than handling the exception. An // alternative would be to re-read the file for every other storage provider as well but // that would be wasteful. From bf14403f3e70b0bcd7222240096119242728d6c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Mon, 8 Apr 2024 13:48:43 +0200 Subject: [PATCH 5/5] Grammar Co-authored-by: Hisham Bin Ateya --- src/docs/reference/modules/Media.AmazonS3/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docs/reference/modules/Media.AmazonS3/README.md b/src/docs/reference/modules/Media.AmazonS3/README.md index f2d8e003638..678c47683b1 100644 --- a/src/docs/reference/modules/Media.AmazonS3/README.md +++ b/src/docs/reference/modules/Media.AmazonS3/README.md @@ -156,7 +156,7 @@ The `BucketName` property and the `BasePath` property are the only templatable p ## Configuring a Local Emulator -During development, instead of using an online S3 resource, you can use a local storage emulator too. This is especially important for development teams, since you don't want to step on each others' feet by using a shared storage. +During development, instead of using an online S3 resource, you can use a local storage emulator too. This is especially important for development teams since you don't want to step on each others' feet by using a shared storage. For emulators, you'll need to configure a `ServiceURL`. Instead of the default [virtual host addressing of buckets](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html), you'll also need to enable path-style addressing (i.e. instead of `http://mybucket.localhost`, the local bucket will be accessible under `http://localhost/mybucket`).