From b256a12bfcb8a58ba55b8d7f8ad9abcff4c26b24 Mon Sep 17 00:00:00 2001 From: Phill Morton Date: Thu, 28 Sep 2023 02:54:42 +0100 Subject: [PATCH 1/5] add environment variable token replacement --- .../Model/Application.cs | 3 +- src/Microsoft.Tye.Hosting/TokenReplacement.cs | 64 +++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Tye.Hosting/Model/Application.cs b/src/Microsoft.Tye.Hosting/Model/Application.cs index 49453aeee..47e1eeb6d 100644 --- a/src/Microsoft.Tye.Hosting/Model/Application.cs +++ b/src/Microsoft.Tye.Hosting/Model/Application.cs @@ -50,9 +50,10 @@ public void PopulateEnvironment(Service service, Action set, str // Inject normal configuration foreach (var pair in service.Description.Configuration) { + if (pair.Value is object) { - set(pair.Name, pair.Value); + set(pair.Name, TokenReplacement.ReplaceEnvironmentValues(pair.Value, bindings)); } else if (pair.Source is object) { diff --git a/src/Microsoft.Tye.Hosting/TokenReplacement.cs b/src/Microsoft.Tye.Hosting/TokenReplacement.cs index 97c788648..94c204be6 100644 --- a/src/Microsoft.Tye.Hosting/TokenReplacement.cs +++ b/src/Microsoft.Tye.Hosting/TokenReplacement.cs @@ -12,6 +12,29 @@ namespace Microsoft.Tye.Hosting { internal static class TokenReplacement { + public static string ReplaceEnvironmentValues(string text, List bindings) + { + var tokens = GetTokens(text); + foreach (var token in tokens) + { + var selectedBinding = ResolveEnvironmentToken(token, bindings); + if (selectedBinding is null) + { + throw new InvalidOperationException($"No available substitutions found for token '{token}'."); + } + + text = selectedBinding.Value.Item2; + var selectedBindingtokens = GetTokens(selectedBinding.Value.Item2); + foreach (var bindingtoken in selectedBindingtokens) + { + var replacement = ReplaceValues(bindingtoken,selectedBinding.Value.Item1, bindings); + text = text.Replace(bindingtoken, replacement); + } + + } + + return text; + } public static string ReplaceValues(string text, EffectiveBinding binding, List bindings) { var tokens = GetTokens(text); @@ -58,6 +81,47 @@ private static HashSet GetTokens(string text) return tokens; } + + private static (EffectiveBinding, string?)? ResolveEnvironmentToken(string token, List bindings) + { + // The language we support for tokens is meant to be pretty DRY. It supports a few different formats: + // + // - ${host}: only allowed inside a connection string, it can refer to the binding. + // - ${env:SOME_VAR}: allowed anywhere. It can refer to any environment variable defined for *this* service. + // - ${service:myservice:port}: allowed anywhere. It can refer to the protocol/host/port of bindings. + + var keys = token[2..^1].Split(':'); + if (keys.Length == 3 && keys[0] == "service") + { + var binding = bindings.FirstOrDefault(b => b.Service == keys[1] && b.Name == null)!; + return (binding, GetValueFromBinding(binding, keys[2])); + } + else if (keys.Length == 4 && keys[0] == "service") + { + var binding = bindings.FirstOrDefault(b => b.Service == keys[1] && b.Name == keys[2])!; + return (binding, GetValueFromBinding(binding, keys[3])); + } + + return null; + + string? GetValueFromBinding(EffectiveBinding binding, string key) + { + return key switch + { + "protocol" => binding.Protocol, + "host" => binding.Host, + "port" => binding.Port?.ToString(CultureInfo.InvariantCulture), + "connectionString" => binding.ConnectionString, + _ => null, + }; + } + + static string? GetEnvironmentVariable(EffectiveBinding binding, string key) + { + var envVar = binding.Env.FirstOrDefault(e => string.Equals(e.Name, key, StringComparison.OrdinalIgnoreCase)); + return envVar?.Value; + } + } private static string? ResolveToken(string token, EffectiveBinding binding, List bindings) { From 871bf22850645d2626b9163b91607d7ccba4b144 Mon Sep 17 00:00:00 2001 From: Phill Morton Date: Thu, 28 Sep 2023 03:16:39 +0100 Subject: [PATCH 2/5] add e2e test --- test/E2ETest/TyeRunTests.cs | 35 ++++++++++++++ .../single-project.sln | 34 ++++++++++++++ .../test-project/Program.cs | 30 ++++++++++++ .../Properties/launchSettings.json | 27 +++++++++++ .../test-project/Startup.cs | 47 +++++++++++++++++++ .../test-project/test-project.csproj | 8 ++++ .../environment-variable-replacement/tye.yaml | 23 +++++++++ 7 files changed, 204 insertions(+) create mode 100644 test/E2ETest/testassets/projects/environment-variable-replacement/single-project.sln create mode 100644 test/E2ETest/testassets/projects/environment-variable-replacement/test-project/Program.cs create mode 100644 test/E2ETest/testassets/projects/environment-variable-replacement/test-project/Properties/launchSettings.json create mode 100644 test/E2ETest/testassets/projects/environment-variable-replacement/test-project/Startup.cs create mode 100644 test/E2ETest/testassets/projects/environment-variable-replacement/test-project/test-project.csproj create mode 100644 test/E2ETest/testassets/projects/environment-variable-replacement/tye.yaml diff --git a/test/E2ETest/TyeRunTests.cs b/test/E2ETest/TyeRunTests.cs index 09c09434e..281b4bd14 100644 --- a/test/E2ETest/TyeRunTests.cs +++ b/test/E2ETest/TyeRunTests.cs @@ -1345,6 +1345,41 @@ public async Task RunWithDotnetEnvVarsDoesNotGetOverriddenByDefaultDotnetEnvVars }); } + [ConditionalFact] + [SkipIfDockerNotRunning] + public async Task RunWithDotnetEnvVarsSelectorForAnotherServiceReplacement() + { + using var projectDirectory = CopyTestProjectDirectory("environment-variable-replacement"); + + var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye.yaml")); + var outputContext = new OutputContext(_sink, Verbosity.Debug); + var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); + + var handler = new HttpClientHandler + { + ServerCertificateCustomValidationCallback = (a, b, c, d) => true, + AllowAutoRedirect = false + }; + + var client = new HttpClient(new RetryHandler(handler)); + + await RunHostingApplication(application, new HostOptions { Docker = true }, async (app, uri) => + { + var backendUri = await GetServiceUrl(client, uri, "test-project"); + + var backendResponse = await client.GetAsync(backendUri); + Assert.True(backendResponse.IsSuccessStatusCode); + + var response = await backendResponse.Content.ReadAsStringAsync(); + var dict = JsonSerializer.Deserialize>(response); + + Assert.Contains(new KeyValuePair("REDIS_CONNECTIONSTRING", "host.docker.internal:6379:mysupersecureP@ssword"), dict); + Assert.Contains(new KeyValuePair("DOTNET_ENVIRONMENT", "dev"), dict); + Assert.Contains(new KeyValuePair("ASPNETCORE_ENVIRONMENT", "dev"), dict); + }); + } + + private async Task GetServiceUrl(HttpClient client, Uri uri, string serviceName) { var serviceResult = await client.GetStringAsync($"{uri}api/v1/services/{serviceName}"); diff --git a/test/E2ETest/testassets/projects/environment-variable-replacement/single-project.sln b/test/E2ETest/testassets/projects/environment-variable-replacement/single-project.sln new file mode 100644 index 000000000..b6c03cfbc --- /dev/null +++ b/test/E2ETest/testassets/projects/environment-variable-replacement/single-project.sln @@ -0,0 +1,34 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.26124.0 +MinimumVisualStudioVersion = 15.0.26124.0 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "test-project", "test-project\test-project.csproj", "{7D3606B2-7B8E-4ABB-BE0A-E0B18285D8F5}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {7D3606B2-7B8E-4ABB-BE0A-E0B18285D8F5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7D3606B2-7B8E-4ABB-BE0A-E0B18285D8F5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7D3606B2-7B8E-4ABB-BE0A-E0B18285D8F5}.Debug|x64.ActiveCfg = Debug|Any CPU + {7D3606B2-7B8E-4ABB-BE0A-E0B18285D8F5}.Debug|x64.Build.0 = Debug|Any CPU + {7D3606B2-7B8E-4ABB-BE0A-E0B18285D8F5}.Debug|x86.ActiveCfg = Debug|Any CPU + {7D3606B2-7B8E-4ABB-BE0A-E0B18285D8F5}.Debug|x86.Build.0 = Debug|Any CPU + {7D3606B2-7B8E-4ABB-BE0A-E0B18285D8F5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7D3606B2-7B8E-4ABB-BE0A-E0B18285D8F5}.Release|Any CPU.Build.0 = Release|Any CPU + {7D3606B2-7B8E-4ABB-BE0A-E0B18285D8F5}.Release|x64.ActiveCfg = Release|Any CPU + {7D3606B2-7B8E-4ABB-BE0A-E0B18285D8F5}.Release|x64.Build.0 = Release|Any CPU + {7D3606B2-7B8E-4ABB-BE0A-E0B18285D8F5}.Release|x86.ActiveCfg = Release|Any CPU + {7D3606B2-7B8E-4ABB-BE0A-E0B18285D8F5}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/test/E2ETest/testassets/projects/environment-variable-replacement/test-project/Program.cs b/test/E2ETest/testassets/projects/environment-variable-replacement/test-project/Program.cs new file mode 100644 index 000000000..3aefed92d --- /dev/null +++ b/test/E2ETest/testassets/projects/environment-variable-replacement/test-project/Program.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace test_project +{ + public class Program + { + public static void Main(string[] args) + { + CreateHostBuilder(args).Build().Run(); + } + + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureWebHostDefaults(webBuilder => + { + webBuilder.UseStartup(); + }); + } +} diff --git a/test/E2ETest/testassets/projects/environment-variable-replacement/test-project/Properties/launchSettings.json b/test/E2ETest/testassets/projects/environment-variable-replacement/test-project/Properties/launchSettings.json new file mode 100644 index 000000000..990224aca --- /dev/null +++ b/test/E2ETest/testassets/projects/environment-variable-replacement/test-project/Properties/launchSettings.json @@ -0,0 +1,27 @@ +{ + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:18482", + "sslPort": 44344 + } + }, + "profiles": { + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "test_project": { + "commandName": "Project", + "launchBrowser": true, + "applicationUrl": "https://localhost:5001;http://localhost:5000", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/test/E2ETest/testassets/projects/environment-variable-replacement/test-project/Startup.cs b/test/E2ETest/testassets/projects/environment-variable-replacement/test-project/Startup.cs new file mode 100644 index 000000000..6565092e3 --- /dev/null +++ b/test/E2ETest/testassets/projects/environment-variable-replacement/test-project/Startup.cs @@ -0,0 +1,47 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace test_project +{ + public class Startup + { + // This method gets called by the runtime. Use this method to add services to the container. + // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 + public void ConfigureServices(IServiceCollection services) + { + } + + // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. + public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + { + if (env.IsDevelopment()) + { + app.UseDeveloperExceptionPage(); + } + + app.UseRouting(); + + app.UseEndpoints(endpoints => + { + endpoints.MapGet("/", + () => Results.Json(new Dictionary + { + { "DOTNET_ENVIRONMENT", Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT") }, + { "ASPNETCORE_ENVIRONMENT", Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") }, + { "REDIS_CONNECTIONSTRING", Environment.GetEnvironmentVariable("REDIS_CONNECTIONSTRING") } + })); + }); + } + } +} diff --git a/test/E2ETest/testassets/projects/environment-variable-replacement/test-project/test-project.csproj b/test/E2ETest/testassets/projects/environment-variable-replacement/test-project/test-project.csproj new file mode 100644 index 000000000..4d2984aac --- /dev/null +++ b/test/E2ETest/testassets/projects/environment-variable-replacement/test-project/test-project.csproj @@ -0,0 +1,8 @@ + + + + net6.0 + test_project + + + diff --git a/test/E2ETest/testassets/projects/environment-variable-replacement/tye.yaml b/test/E2ETest/testassets/projects/environment-variable-replacement/tye.yaml new file mode 100644 index 000000000..e3ec56209 --- /dev/null +++ b/test/E2ETest/testassets/projects/environment-variable-replacement/tye.yaml @@ -0,0 +1,23 @@ +# tye application configuration file +# read all about it at https://github.com/dotnet/tye +name: dotnet-env-replacement +dashboardPort: 8090 +services: +- name: test-project + project: test-project/test-project.csproj + env: + - name: DOTNET_ENVIRONMENT + value: "dev" + - name: ASPNETCORE_ENVIRONMENT + value: "dev" + - name: REDIS_CONNECTIONSTRING + value: "${service:redis:connectionString}" +- name: redis + image: redis + env: + - name: RANDOMPASSWORD + value: "mysupersecureP@ssword" + bindings: + - port: 6379 + connectionString: ${host}:${port}:${env:RANDOMPASSWORD} #I know not a real password, but I just want to prove the relacement + From 397c12f6292b22419e974053c760c320b16f65e0 Mon Sep 17 00:00:00 2001 From: Phill Morton Date: Thu, 28 Sep 2023 03:19:46 +0100 Subject: [PATCH 3/5] use alpine in E2E --- src/Microsoft.Tye.Hosting/TokenReplacement.cs | 11 +++++++---- .../environment-variable-replacement/tye.yaml | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.Tye.Hosting/TokenReplacement.cs b/src/Microsoft.Tye.Hosting/TokenReplacement.cs index 94c204be6..2c6db2625 100644 --- a/src/Microsoft.Tye.Hosting/TokenReplacement.cs +++ b/src/Microsoft.Tye.Hosting/TokenReplacement.cs @@ -23,11 +23,13 @@ public static string ReplaceEnvironmentValues(string text, List bindings) { var tokens = GetTokens(text); @@ -82,7 +85,7 @@ private static HashSet GetTokens(string text) return tokens; } - private static (EffectiveBinding, string?)? ResolveEnvironmentToken(string token, List bindings) + private static (EffectiveBinding binding, string? text)? ResolveEnvironmentToken(string token, List bindings) { // The language we support for tokens is meant to be pretty DRY. It supports a few different formats: // diff --git a/test/E2ETest/testassets/projects/environment-variable-replacement/tye.yaml b/test/E2ETest/testassets/projects/environment-variable-replacement/tye.yaml index e3ec56209..b7bccd9bd 100644 --- a/test/E2ETest/testassets/projects/environment-variable-replacement/tye.yaml +++ b/test/E2ETest/testassets/projects/environment-variable-replacement/tye.yaml @@ -13,11 +13,11 @@ services: - name: REDIS_CONNECTIONSTRING value: "${service:redis:connectionString}" - name: redis - image: redis + image: alpine # just need a tiny image to make the test run faster env: - name: RANDOMPASSWORD value: "mysupersecureP@ssword" bindings: - port: 6379 - connectionString: ${host}:${port}:${env:RANDOMPASSWORD} #I know not a real password, but I just want to prove the relacement + connectionString: ${host}:${port}:${env:RANDOMPASSWORD} #I know not a real password, but I just want to prove the replacement From ec23b9134e350378e216b3db914bb7e3047a753d Mon Sep 17 00:00:00 2001 From: Phill Morton Date: Thu, 28 Sep 2023 10:01:22 +0100 Subject: [PATCH 4/5] use password that is excluded in secret scanning --- .../projects/environment-variable-replacement/tye.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/E2ETest/testassets/projects/environment-variable-replacement/tye.yaml b/test/E2ETest/testassets/projects/environment-variable-replacement/tye.yaml index b7bccd9bd..25113746c 100644 --- a/test/E2ETest/testassets/projects/environment-variable-replacement/tye.yaml +++ b/test/E2ETest/testassets/projects/environment-variable-replacement/tye.yaml @@ -16,7 +16,7 @@ services: image: alpine # just need a tiny image to make the test run faster env: - name: RANDOMPASSWORD - value: "mysupersecureP@ssword" + value: "pass@word1" bindings: - port: 6379 connectionString: ${host}:${port}:${env:RANDOMPASSWORD} #I know not a real password, but I just want to prove the replacement From 9a416c37331a5abed120f40423db1e92a107360b Mon Sep 17 00:00:00 2001 From: Phill Morton Date: Thu, 28 Sep 2023 10:01:39 +0100 Subject: [PATCH 5/5] Apply feedback from github build --- src/Microsoft.Tye.Hosting/TokenReplacement.cs | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/Microsoft.Tye.Hosting/TokenReplacement.cs b/src/Microsoft.Tye.Hosting/TokenReplacement.cs index 2c6db2625..a4def8122 100644 --- a/src/Microsoft.Tye.Hosting/TokenReplacement.cs +++ b/src/Microsoft.Tye.Hosting/TokenReplacement.cs @@ -22,19 +22,18 @@ public static string ReplaceEnvironmentValues(string text, List null, }; } - - static string? GetEnvironmentVariable(EffectiveBinding binding, string key) - { - var envVar = binding.Env.FirstOrDefault(e => string.Equals(e.Name, key, StringComparison.OrdinalIgnoreCase)); - return envVar?.Value; - } } private static string? ResolveToken(string token, EffectiveBinding binding, List bindings)