diff --git a/Azure.Functions.Cli.sln b/Azure.Functions.Cli.sln index f4fe46ec2..018d78c66 100644 --- a/Azure.Functions.Cli.sln +++ b/Azure.Functions.Cli.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.29519.87 +# Visual Studio Version 17 +VisualStudioVersion = 17.12.35506.116 d17.12 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{5F51C958-39C0-4E0C-9165-71D0BCE647BC}" EndProject @@ -11,6 +11,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Functions.Cli", "src\ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Functions.Cli.Tests", "test\Azure.Functions.Cli.Tests\Azure.Functions.Cli.Tests.csproj", "{EAEA6EDB-A301-4A50-86D8-91859DABE30E}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ZippedExe", "test\ZippedExe\ZippedExe.csproj", "{2CD45039-0ABD-4082-87D0-52BB5D467B50}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -25,6 +27,10 @@ Global {EAEA6EDB-A301-4A50-86D8-91859DABE30E}.Debug|Any CPU.Build.0 = Debug|Any CPU {EAEA6EDB-A301-4A50-86D8-91859DABE30E}.Release|Any CPU.ActiveCfg = Release|Any CPU {EAEA6EDB-A301-4A50-86D8-91859DABE30E}.Release|Any CPU.Build.0 = Release|Any CPU + {2CD45039-0ABD-4082-87D0-52BB5D467B50}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2CD45039-0ABD-4082-87D0-52BB5D467B50}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2CD45039-0ABD-4082-87D0-52BB5D467B50}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2CD45039-0ABD-4082-87D0-52BB5D467B50}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -32,6 +38,7 @@ Global GlobalSection(NestedProjects) = preSolution {6608738C-3BDB-47F5-BC62-66A8BDF9D884} = {5F51C958-39C0-4E0C-9165-71D0BCE647BC} {EAEA6EDB-A301-4A50-86D8-91859DABE30E} = {6EE1D011-2334-44F2-9D41-608B969DAE6D} + {2CD45039-0ABD-4082-87D0-52BB5D467B50} = {6EE1D011-2334-44F2-9D41-608B969DAE6D} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {FA1E01D6-A57B-4061-A333-EDC511D283C0} diff --git a/build/Program.cs b/build/Program.cs index 261372b61..13c374fd4 100644 --- a/build/Program.cs +++ b/build/Program.cs @@ -1,5 +1,4 @@ -using System.IO; -using System.Linq; +using System.Linq; using System.Net; using static Build.BuildSteps; diff --git a/eng/ci/templates/public/jobs/build-test-public.yml b/eng/ci/templates/public/jobs/build-test-public.yml index bc5ffeb69..a741ff433 100644 --- a/eng/ci/templates/public/jobs/build-test-public.yml +++ b/eng/ci/templates/public/jobs/build-test-public.yml @@ -56,4 +56,31 @@ jobs: testResultsFormat: 'VSTest' testResultsFiles: '**/*.trx' failTaskOnFailedTests: true - condition: succeededOrFailed() \ No newline at end of file + condition: succeededOrFailed() + templateContext: + outputs: + - output: pipelineArtifact + path: 'test/Azure.Functions.Cli.Tests/bin/Debug/net8.0/ZippedOnWindows.zip' + artifact: ZippedOnWindows + +- job: Test_Linux + timeoutInMinutes: "180" + pool: + name: 1es-pool-azfunc-public + image: '1es-ubuntu-22.04' + os: 'linux' + steps: + - download: current + displayName: 'Download test zip' + artifact: ZippedOnWindows + - script: | + sudo apt-get update + sudo apt-get -y install fuse-zip + displayName: 'Install fuse-zip' + - task: DotNetCoreCLI@2 + inputs: + command: 'test' + projects: '**/Azure.Functions.Cli.Tests.csproj' + arguments: '--filter CreateZip_Succeeds' + displayName: 'Run zip test' + dependsOn: Default \ No newline at end of file diff --git a/skipPackagesCve.json b/skipPackagesCve.json index 57251b7f7..19bdfa9b2 100644 --- a/skipPackagesCve.json +++ b/skipPackagesCve.json @@ -1,5 +1,4 @@ { "packages": [ - "DotNetZip" ] } \ No newline at end of file diff --git a/src/Azure.Functions.Cli/Actions/AzureActions/PublishFunctionAppAction.cs b/src/Azure.Functions.Cli/Actions/AzureActions/PublishFunctionAppAction.cs index a3f1a18df..163b7b249 100644 --- a/src/Azure.Functions.Cli/Actions/AzureActions/PublishFunctionAppAction.cs +++ b/src/Azure.Functions.Cli/Actions/AzureActions/PublishFunctionAppAction.cs @@ -169,7 +169,7 @@ public override async Task RunAsync() var targetFramework = await DotnetHelpers.DetermineTargetFramework(Path.GetDirectoryName(projectFilePath)); var majorDotnetVersion = StacksApiHelper.GetMajorDotnetVersionFromDotnetVersionInProject(targetFramework); - + if (majorDotnetVersion != null) { // Get Stacks @@ -179,8 +179,8 @@ public override async Task RunAsync() ShowEolMessage(stacks, runtimeSettings, majorDotnetVersion.Value); // This is for future proofing with stacks API for future dotnet versions. - if (runtimeSettings != null && - (runtimeSettings.IsDeprecated == null || runtimeSettings.IsDeprecated == false) && + if (runtimeSettings != null && + (runtimeSettings.IsDeprecated == null || runtimeSettings.IsDeprecated == false) && (runtimeSettings.IsDeprecatedForRuntime == null || runtimeSettings.IsDeprecatedForRuntime == false)) { _requiredNetFrameworkVersion = $"{majorDotnetVersion}.0"; @@ -275,12 +275,12 @@ private async Task> ValidateFunctionAppPublish(Site workerRuntimeStr = functionApp.FunctionAppConfig.runtime.name; } - if (workerRuntime == WorkerRuntime.None) - { + if (workerRuntime == WorkerRuntime.None) + { throw new CliException($"Worker runtime is not set. Please set a valid runtime using {Constants.FunctionsWorkerRuntime}"); } - if ((functionApp.IsFlex && !string.IsNullOrEmpty(workerRuntimeStr) || + if ((functionApp.IsFlex && !string.IsNullOrEmpty(workerRuntimeStr) || (!functionApp.IsFlex && functionApp.AzureAppSettings.TryGetValue(Constants.FunctionsWorkerRuntime, out workerRuntimeStr)))) { var resolution = $"You can pass --force to update your Azure app with '{workerRuntime}' as a '{Constants.FunctionsWorkerRuntime}'"; @@ -350,7 +350,7 @@ private async Task> ValidateFunctionAppPublish(Site (functionApp.IsFlex && !PythonHelpers.IsFlexPythonRuntimeVersionMatched(functionApp.FunctionAppConfig?.runtime?.name, functionApp.FunctionAppConfig?.runtime?.version, localVersion.Major, localVersion.Minor))) { ColoredConsole.WriteLine(WarningColor($"Local python version '{localVersion.Version}' is different from the version expected for your deployed Function App." + - $" This may result in 'ModuleNotFound' errors in Azure Functions. Please create a Python Function App for version {localVersion.Major}.{localVersion.Minor} or change the virtual environment on your local machine to match '{(functionApp.IsFlex? functionApp.FunctionAppConfig.runtime.version: functionApp.LinuxFxVersion)}'.")); + $" This may result in 'ModuleNotFound' errors in Azure Functions. Please create a Python Function App for version {localVersion.Major}.{localVersion.Minor} or change the virtual environment on your local machine to match '{(functionApp.IsFlex ? functionApp.FunctionAppConfig.runtime.version : functionApp.LinuxFxVersion)}'.")); } } @@ -559,6 +559,9 @@ private async Task PublishFunctionApp(Site functionApp, GitIgnoreParser ignorePa ColoredConsole.WriteLine(WarningColor("Recommend using '--build remote' to resolve project dependencies remotely on Azure")); } + bool useGoZip = EnvironmentHelper.GetEnvironmentVariableAsBool(Constants.UseGoZip); + TelemetryHelpers.AddCommandEventToDictionary(TelemetryCommandEvents, "UseGoZip", useGoZip.ToString()); + ColoredConsole.WriteLine(GetLogMessage("Starting the function app deployment...")); Func> zipStreamFactory = () => ZipHelper.GetAppZipFile(functionAppRoot, BuildNativeDeps, PublishBuildOption, NoBuild, ignoreParser, AdditionalPackages, ignoreDotNetCheck: true); diff --git a/src/Azure.Functions.Cli/Actions/LocalActions/PackAction.cs b/src/Azure.Functions.Cli/Actions/LocalActions/PackAction.cs index ddf538391..21a84e2b2 100644 --- a/src/Azure.Functions.Cli/Actions/LocalActions/PackAction.cs +++ b/src/Azure.Functions.Cli/Actions/LocalActions/PackAction.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; @@ -9,7 +8,6 @@ using Colors.Net; using Fclp; using Microsoft.Azure.WebJobs.Script; -using static Colors.Net.StringStaticMethods; using static Azure.Functions.Cli.Common.OutputTheme; namespace Azure.Functions.Cli.Actions.LocalActions @@ -104,6 +102,10 @@ public override async Task RunAsync() // Restore all valid extensions var installExtensionAction = new InstallExtensionAction(_secretsManager, false); await installExtensionAction.RunAsync(); + + bool useGoZip = EnvironmentHelper.GetEnvironmentVariableAsBool(Constants.UseGoZip); + TelemetryHelpers.AddCommandEventToDictionary(TelemetryCommandEvents, "UseGoZip", useGoZip.ToString()); + var stream = await ZipHelper.GetAppZipFile(functionAppRoot, BuildNativeDeps, noBuild: false, buildOption: BuildOption.Default, additionalPackages: AdditionalPackages); if (Squashfs) diff --git a/src/Azure.Functions.Cli/Azure.Functions.Cli.csproj b/src/Azure.Functions.Cli/Azure.Functions.Cli.csproj index 92da05a4e..ac7d34bf4 100644 --- a/src/Azure.Functions.Cli/Azure.Functions.Cli.csproj +++ b/src/Azure.Functions.Cli/Azure.Functions.Cli.csproj @@ -275,7 +275,6 @@ - @@ -284,8 +283,7 @@ - - + diff --git a/src/Azure.Functions.Cli/Common/Constants.cs b/src/Azure.Functions.Cli/Common/Constants.cs index 2880cb70a..83f80bee3 100644 --- a/src/Azure.Functions.Cli/Common/Constants.cs +++ b/src/Azure.Functions.Cli/Common/Constants.cs @@ -1,7 +1,7 @@ -using Azure.Functions.Cli.Helpers; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Reflection; +using Azure.Functions.Cli.Helpers; namespace Azure.Functions.Cli.Common { @@ -63,6 +63,7 @@ internal static class Constants public const string UserSecretsIdElementName = "UserSecretsId"; public const string TargetFrameworkElementName = "TargetFramework"; public const string DisplayLogo = "FUNCTIONS_CORE_TOOLS_DISPLAY_LOGO"; + public const string UseGoZip = "FUNCTIONS_CORE_TOOLS_USE_GOZIP"; public const string AspNetCoreSupressStatusMessages = "ASPNETCORE_SUPPRESSSTATUSMESSAGES"; public const string SequentialJobHostRestart = "AzureFunctionsJobHost__SequentialRestart"; public const long DefaultMaxRequestBodySize = 104857600; diff --git a/src/Azure.Functions.Cli/Helpers/ZipHelper.cs b/src/Azure.Functions.Cli/Helpers/ZipHelper.cs index 88377aa85..1457e1ebc 100644 --- a/src/Azure.Functions.Cli/Helpers/ZipHelper.cs +++ b/src/Azure.Functions.Cli/Helpers/ZipHelper.cs @@ -1,15 +1,14 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using System.Reflection; +using System.Text; using System.Threading.Tasks; using Azure.Functions.Cli.Common; using Colors.Net; -using Ionic.Zip; using static Colors.Net.StringStaticMethods; -using static Azure.Functions.Cli.Common.OutputTheme; -using System.Text; namespace Azure.Functions.Cli.Helpers { @@ -26,10 +25,12 @@ public static async Task GetAppZipFile(string functionAppRoot, bool buil if (noBuild) { ColoredConsole.WriteLine(DarkYellow("Skipping build event for functions project (--no-build).")); - } else if (buildOption == BuildOption.Remote) + } + else if (buildOption == BuildOption.Remote) { ColoredConsole.WriteLine(DarkYellow("Performing remote build for functions project.")); - } else if (buildOption == BuildOption.Local) + } + else if (buildOption == BuildOption.Local) { ColoredConsole.WriteLine(DarkYellow("Performing local build for functions project.")); } @@ -51,7 +52,7 @@ public static async Task GetAppZipFile(string functionAppRoot, bool buil { var customHandler = await HostHelpers.GetCustomHandlerExecutable(); IEnumerable executables = !string.IsNullOrEmpty(customHandler) - ? new [] {customHandler} + ? new[] { customHandler } : Enumerable.Empty(); return await CreateZip(FileSystemHelpers.GetLocalFiles(functionAppRoot, ignoreParser, false), functionAppRoot, executables); } @@ -59,17 +60,23 @@ public static async Task GetAppZipFile(string functionAppRoot, bool buil public static async Task CreateZip(IEnumerable files, string rootPath, IEnumerable executables) { - var zipFilePath = Path.GetTempFileName(); + // temporarily provide an escape hatch to use gozip in case there are bugs in the dotnet implementation + bool useGoZip = EnvironmentHelper.GetEnvironmentVariableAsBool(Constants.UseGoZip); - if (GoZipExists(out string goZipLocation)) + if (useGoZip) { - return await CreateGoZip(files, rootPath, zipFilePath, goZipLocation, executables); - } + if (GoZipExists(out string goZipLocation)) + { + ColoredConsole.WriteLine(DarkYellow("Using gozip for packaging.")); + var zipFilePath = Path.GetTempFileName(); + return await CreateGoZip(files, rootPath, zipFilePath, goZipLocation, executables); + } - ColoredConsole.WriteLine(Yellow("Could not find gozip for packaging. Using DotNetZip to package. " + - "This may cause problems preserving file permissions when using in a Linux based environment.")); + ColoredConsole.WriteLine(Yellow("Could not find gozip for packaging. Using DotNetZip to package. " + + "This may cause problems preserving file permissions when using in a Linux based environment.")); + } - return CreateDotNetZip(files, rootPath, zipFilePath); + return CreateDotNetZip(files, rootPath, executables); } public static bool GoZipExists(out string fileLocation) @@ -87,21 +94,122 @@ public static bool GoZipExists(out string fileLocation) return false; } - public static Stream CreateDotNetZip(IEnumerable files, string rootPath, string zipFilePath) + public static Stream CreateDotNetZip(IEnumerable files, string rootPath, IEnumerable executables) { - const int defaultBufferSize = 4096; - var fileStream = new FileStream(zipFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, defaultBufferSize, FileOptions.DeleteOnClose); - using (ZipFile zip = new ZipFile()) + // See section 4.4.2.2 in the zip spec: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT + const byte CreatedByUnix = 3; + + // Signature that defines beginning of each file metadata. See section 4.3.12 in the zip spec above. + const uint centralDirectorySignature = 0x02014B50; + + // Unix file permissions + int UnixExecutablePermissions = Convert.ToInt32("100777", 8) << 16; + int UnixReadWritePermissions = Convert.ToInt32("100666", 8) << 16; + + var memStream = new MemoryStream(); + + using (var zip = new ZipArchive(memStream, ZipArchiveMode.Create, leaveOpen: true)) { - zip.CompressionLevel = Ionic.Zlib.CompressionLevel.BestSpeed; foreach (var file in files) { - zip.AddFile(file.FixFileNameForZip(rootPath)); + var entryName = file.FixFileNameForZip(rootPath); + var entry = zip.CreateEntryFromFile(file, entryName); + + entry.ExternalAttributes = executables.Contains(entryName) ? UnixExecutablePermissions : UnixReadWritePermissions; + } + } + + // In order to properly mount and/or unzip this in Azure, we need to create the zip as if it were + // Unix so that the correct file permissions set above are applied. To do this, we walk backwards + // through the stream and update the "created by" field to 3, which indicates it was created by Unix. + if (OperatingSystem.IsWindows()) + { + memStream.Seek(0, SeekOrigin.End); + + // Update the file header in the zip file for every file to indicate that it was created by Unix + while (SeekBackwardsToSignature(memStream, centralDirectorySignature)) + { + // The field we need to set is 5 bytes from the beginning of the signature. Set it, + // then move back to the previous location so we can continue. + memStream.Seek(5, SeekOrigin.Current); + memStream.WriteByte(CreatedByUnix); + memStream.Seek(-6, SeekOrigin.Current); + } + } + + memStream.Seek(0, SeekOrigin.Begin); + return memStream; + } + + // Assumes all bytes of signatureToFind are non zero. + // If the signature is found then returns true and positions stream at first byte of signature + // If the signature is not found, returns false + private static bool SeekBackwardsToSignature(Stream stream, uint signatureToFind) + { + int bufferPointer = 0; + uint currentSignature = 0; + // 32-byte buffer is arbitrary and is following the runtime implementation here: + // https://github.com/dotnet/runtime/blob/ea97babd7ccfd2f6e9553093d315f26b51e4c7ac/src/libraries/System.IO.Compression/src/System/IO/Compression/ZipHelper.cs#L16 + byte[] buffer = new byte[32]; + + bool outOfBytes = false; + bool signatureFound = false; + + while (!signatureFound && !outOfBytes) + { + outOfBytes = SeekBackwardsAndRead(stream, buffer, out bufferPointer); + + while (bufferPointer >= 0 && !signatureFound) + { + currentSignature = (currentSignature << 8) | ((uint)buffer[bufferPointer]); + if (currentSignature == signatureToFind) + { + signatureFound = true; + break; + } + else + { + bufferPointer--; + } } - zip.Save(fileStream); } - fileStream.Seek(0, SeekOrigin.Begin); - return fileStream; + + if (!signatureFound) + { + return false; + } + else + { + // set the stream up to continue from here next call + stream.Seek(bufferPointer, SeekOrigin.Current); + return true; + } + } + + // Returns true if we are out of bytes + // This method (and SeekBackwardsToSignature) are mostly copied from the runtime implementation here: + // https://github.com/dotnet/runtime/blob/ea97babd7ccfd2f6e9553093d315f26b51e4c7ac/src/libraries/System.IO.Compression/src/System/IO/Compression/ZipHelper.cs#L172-L191 + private static bool SeekBackwardsAndRead(Stream stream, byte[] buffer, out int bufferPointer) + { + if (stream.Position >= buffer.Length) + { + stream.Seek(-buffer.Length, SeekOrigin.Current); + stream.ReadExactly(buffer.AsSpan()); + stream.Seek(-buffer.Length, SeekOrigin.Current); + bufferPointer = buffer.Length - 1; + return false; + } + else + { + // if we cannot fill the buffer, read everything that's left and + // return back that position in the buffer to the caller + int bytesToRead = (int)stream.Position; + stream.Seek(0, SeekOrigin.Begin); + stream.ReadExactly(buffer, 0, bytesToRead); + stream.Seek(0, SeekOrigin.Begin); + bufferPointer = bytesToRead - 1; + return true; + } } public static async Task CreateGoZip(IEnumerable files, string rootPath, string zipFilePath, string goZipLocation, IEnumerable executables) diff --git a/test/Azure.Functions.Cli.Tests/E2E/Helpers/ProcessHelper.cs b/test/Azure.Functions.Cli.Tests/E2E/Helpers/ProcessHelper.cs index 909f4b93b..50a465842 100644 --- a/test/Azure.Functions.Cli.Tests/E2E/Helpers/ProcessHelper.cs +++ b/test/Azure.Functions.Cli.Tests/E2E/Helpers/ProcessHelper.cs @@ -94,34 +94,75 @@ public static int GetAvailablePort() } } - private static string ExecuteCommand(string command) + public static void RunProcess(string fileName, string arguments, string workingDirectory, Action writeOutput = null, Action writeError = null) { - using (Process p = new Process()) - { - string commandOut = string.Empty; + TimeSpan procTimeout = TimeSpan.FromMinutes(3); - p.StartInfo.FileName = CommandExe; - p.StartInfo.Arguments = command; - p.StartInfo.UseShellExecute = false; - p.StartInfo.CreateNoWindow = true; - p.StartInfo.RedirectStandardOutput = true; - p.StartInfo.RedirectStandardError = true; - p.Start(); + ProcessStartInfo startInfo = new() + { + CreateNoWindow = true, + UseShellExecute = false, + WorkingDirectory = workingDirectory, + RedirectStandardOutput = true, + RedirectStandardError = true, + FileName = fileName + }; + + if (!string.IsNullOrEmpty(arguments)) + { + startInfo.Arguments = arguments; + } - commandOut = p.StandardOutput.ReadToEnd(); - string errors = p.StandardError.ReadToEnd(); + using Process testProcess = new() + { + StartInfo = startInfo, + }; - try + testProcess.OutputDataReceived += (sender, e) => + { + if (e.Data != null) { - p.WaitForExit(TimeSpan.FromSeconds(2).Milliseconds); + writeOutput?.Invoke(e.Data); } - catch (Exception exp) + }; + + testProcess.ErrorDataReceived += (sender, e) => + { + if (e.Data != null) { - Console.WriteLine(exp.ToString()); + writeError?.Invoke(e.Data); } + }; - return commandOut; + testProcess.Start(); + testProcess.BeginOutputReadLine(); + testProcess.BeginErrorReadLine(); + + bool completed = false; + try + { + completed = testProcess.WaitForExit((int)procTimeout.TotalMilliseconds); } + catch (Exception ex) + { + Console.WriteLine($"Process '{fileName} {arguments}' in working directory '{workingDirectory}' threw exception '{ex}'."); + } + + if (!completed) + { + throw new TimeoutException($"Process '{fileName} {arguments}' in working directory '{workingDirectory}' did not complete in {procTimeout}."); + } + + testProcess.WaitForExit(); + } + + private static string ExecuteCommand(string command) + { + string output = string.Empty; + + RunProcess(CommandExe, command, null, writeOutput: o => output += o + Environment.NewLine); + + return output; } } } diff --git a/test/Azure.Functions.Cli.Tests/E2E/InitTests.cs b/test/Azure.Functions.Cli.Tests/E2E/InitTests.cs index 95dda69d6..40aae960c 100644 --- a/test/Azure.Functions.Cli.Tests/E2E/InitTests.cs +++ b/test/Azure.Functions.Cli.Tests/E2E/InitTests.cs @@ -45,7 +45,7 @@ public Task init_with_worker_runtime(string workerRuntime) return CliTester.Run(new RunConfiguration { - Commands = new[] { $"init . --worker-runtime {workerRuntime}" }, + Commands = new[] { $"init . --worker-runtime {workerRuntime} --skip-npm-install" }, CheckFiles = files.ToArray(), OutputContains = new[] { @@ -303,7 +303,7 @@ public Task init_with_unsupported_target_framework_for_dotnet() }, _output); } - [Fact(Skip="Flaky test")] + [Fact(Skip = "Flaky test")] public Task init_with_no_source_control() { return CliTester.Run(new RunConfiguration @@ -902,12 +902,12 @@ public Task init_python_app_generates_getting_started_md() } [Theory] - [InlineData("dotnet-isolated","4", "net6.0")] + [InlineData("dotnet-isolated", "4", "net6.0")] [InlineData("dotnet-isolated", "4", "net7.0")] - [InlineData("dotnet-isolated","4", "net8.0")] + [InlineData("dotnet-isolated", "4", "net8.0")] public Task init_docker_only_for_existing_project_TargetFramework(string workerRuntime, string version, string TargetFramework) { - var TargetFrameworkstr = TargetFramework.Replace("net", string.Empty); + var TargetFrameworkstr = TargetFramework.Replace("net", string.Empty); return CliTester.Run(new RunConfiguration { Commands = new[] diff --git a/test/Azure.Functions.Cli.Tests/E2E/StartTests.cs b/test/Azure.Functions.Cli.Tests/E2E/StartTests.cs index af0872346..81bb91483 100644 --- a/test/Azure.Functions.Cli.Tests/E2E/StartTests.cs +++ b/test/Azure.Functions.Cli.Tests/E2E/StartTests.cs @@ -1,17 +1,17 @@ using System; -using System.IO; using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Net; using System.Net.Http; +using System.Net.Sockets; using System.Text; using System.Threading.Tasks; +using Azure.Functions.Cli.Common; using Azure.Functions.Cli.Tests.E2E.Helpers; using FluentAssertions; using Xunit; using Xunit.Abstractions; -using System.Net.Sockets; -using System.Net; -using System.Diagnostics; -using Azure.Functions.Cli.Common; namespace Azure.Functions.Cli.Tests.E2E { @@ -976,7 +976,7 @@ await CliTester.Run(new RunConfiguration[] }, Test = async (_, p, stdout) => { - await LogWatcher.WaitForLogOutput(stdout, "Initializing function HTTP routes", TimeSpan.FromSeconds(5)); + await LogWatcher.WaitForLogOutput(stdout, "Worker process started and initialized", TimeSpan.FromSeconds(5)); p.Kill(); }, CommandTimeout = TimeSpan.FromSeconds(300) diff --git a/test/Azure.Functions.Cli.Tests/ZipHelperTests.cs b/test/Azure.Functions.Cli.Tests/ZipHelperTests.cs new file mode 100644 index 000000000..64b5c4a1a --- /dev/null +++ b/test/Azure.Functions.Cli.Tests/ZipHelperTests.cs @@ -0,0 +1,197 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; +using Azure.Functions.Cli.Common; +using Azure.Functions.Cli.Helpers; +using Azure.Functions.Cli.Tests.E2E.Helpers; +using Xunit; +using Xunit.Abstractions; + +namespace Azure.Functions.Cli.Tests +{ + public class ZipHelperTests + { + private readonly ITestOutputHelper _output; + private bool _isCI = Environment.GetEnvironmentVariable("TF_BUILD")?.ToLowerInvariant() == "true"; + + public ZipHelperTests(ITestOutputHelper output) + { + _output = output; + + // reset to default in case other tests have set this up with mocks + FileSystemHelpers.Instance = null; + } + + [Fact] + public async Task CreateZip_Succeeds() + { + var windowsZip = await BuildAndCopyFileToZipAsync("win-x64"); + var linuxZip = await BuildAndCopyFileToZipAsync("linux-x64"); + + if (OperatingSystem.IsWindows()) + { + if (_isCI) + { + // copy the windows-built linux zip so we can include it in ci artifacts for validation on linux + File.Copy(linuxZip, Path.Combine(Directory.GetCurrentDirectory(), "ZippedOnWindows.zip"), true); + } + + VerifyWindowsZip(windowsZip); + } + else if (OperatingSystem.IsLinux()) + { + VerifyLinuxZip(linuxZip); + + if (_isCI) + { + var workspace = Environment.GetEnvironmentVariable("PIPELINE_WORKSPACE"); + Assert.NotNull(workspace); + + // this should only run in CI where we've built a zip on windows and copied it here + var zippedOnWindows = Directory.GetFiles(workspace, "ZippedOnWindows.zip", SearchOption.AllDirectories).Single(); + VerifyLinuxZip(zippedOnWindows); + } + } + else + { + throw new Exception("Unsupported OS"); + } + } + + private async Task BuildAndCopyFileToZipAsync(string rid) + { + // files we'll need to zip up + const string proj = "ZippedExe"; + string exe = rid.StartsWith("linux") ? proj : $"{proj}.exe"; + string dll = $"{proj}.dll"; + string config = $"{proj}.runtimeconfig.json"; + + var tempDir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + Directory.CreateDirectory(tempDir); + + // Create some temp files + var files = new List(); + for (int i = 0; i < 10; i++) + { + var file = Path.Combine(tempDir, Path.GetRandomFileName()); + File.WriteAllText(file, Guid.NewGuid().ToString()); + files.Add(file); + } + + // walk up to the 'test' directory + var dir = new DirectoryInfo(Directory.GetCurrentDirectory()); + dir = dir.Parent.Parent.Parent.Parent; + + // build the project for the rid + var csproj = dir.GetFiles($"{proj}.csproj", SearchOption.AllDirectories).FirstOrDefault(); + var csprojDir = csproj.Directory.FullName; + ProcessHelper.RunProcess("dotnet", $"build -r {rid}", csprojDir, writeOutput: WriteOutput); + + var outPath = Path.Combine(csprojDir, "bin", "Debug", "net8.0", rid); + + // copy the files to the zip dir + foreach (string fileName in new[] { exe, dll, config }) + { + var f = new DirectoryInfo(outPath).GetFiles(fileName, SearchOption.AllDirectories).FirstOrDefault(); + Assert.True(f != null, $"{fileName} not found."); + string destFile = Path.Combine(tempDir, fileName); + File.Copy(f.FullName, destFile); + files.Add(destFile); + } + + // use our zip utilities to zip them + var zipFile = Path.Combine(tempDir, "test.zip"); + var stream = await ZipHelper.CreateZip(files, tempDir, executables: new string[] { exe }); + + await FileSystemHelpers.WriteToFile(zipFile, stream); + + return zipFile; + } + + private static void VerifyWindowsZip(string zipFile) + { + var unzipPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + ZipFile.ExtractToDirectory(zipFile, unzipPath); + + string exeOutput = null; + string exeError = null; + + ProcessHelper.RunProcess(Path.Combine(unzipPath, "ZippedExe.exe"), string.Empty, unzipPath, o => exeOutput += o + Environment.NewLine, e => exeError += e + Environment.NewLine); + + Assert.Null(exeError); + Assert.Equal("Hello, World!", exeOutput.Trim()); + } + + private void VerifyLinuxZip(string zipFile) + { + const string exeName = "ZippedExe"; + List outputLines = new List(); + + void CaptureOutput(string output) + { + outputLines.Add(output); + WriteOutput(output); + } + + var zipFileName = Path.GetFileName(zipFile); + var zipDir = Path.GetDirectoryName(zipFile); + var mntDir = Path.Combine(zipDir, "mnt"); + + Directory.CreateDirectory(mntDir); + + // this is what our hosting environment does; we need to validate we can run the exe when mounted like this + ProcessHelper.RunProcess("fuse-zip", $"./{zipFileName} ./mnt -r", zipDir, writeOutput: WriteOutput); + ProcessHelper.RunProcess("bash", $"-c \"ls -l\"", mntDir, writeOutput: CaptureOutput); + + Assert.Equal(14, outputLines.Count()); + + // ignore first ('total ...') to validate file perms + foreach (string line in outputLines.Skip(1)) + { + // exe should be executable + if (line.EndsWith(exeName)) + { + Assert.StartsWith("-rwxrwxrwx", line); + } + else + { + Assert.StartsWith("-rw-rw-rw-", line); + } + } + + var files = Directory.GetFiles(mntDir, "*.*", SearchOption.AllDirectories); + Assert.Equal(13, files.Length); + foreach (string file in files) + { + var fileInfo = new FileInfo(file); + if (fileInfo.Name == exeName) + { + var readWriteExecute = UnixFileMode.GroupWrite | UnixFileMode.GroupRead | UnixFileMode.GroupExecute | + UnixFileMode.UserWrite | UnixFileMode.UserRead | UnixFileMode.UserExecute | + UnixFileMode.OtherWrite | UnixFileMode.OtherRead | UnixFileMode.OtherExecute; + + Assert.Equal(readWriteExecute, fileInfo.UnixFileMode); + } + else + { + var readWrite = UnixFileMode.GroupWrite | UnixFileMode.GroupRead | UnixFileMode.UserWrite | + UnixFileMode.UserRead | UnixFileMode.OtherWrite | UnixFileMode.OtherRead; + + Assert.Equal(readWrite, fileInfo.UnixFileMode); + } + } + + outputLines.Clear(); + ProcessHelper.RunProcess($"{Path.Combine(mntDir, exeName)}", string.Empty, mntDir, writeOutput: CaptureOutput); + Assert.Equal("Hello, World!", outputLines.Last()); + } + + private void WriteOutput(string output) + { + _output.WriteLine(output); + } + } +} \ No newline at end of file diff --git a/test/ZippedExe/Program.cs b/test/ZippedExe/Program.cs new file mode 100644 index 000000000..cff705ca1 --- /dev/null +++ b/test/ZippedExe/Program.cs @@ -0,0 +1,2 @@ +// Using this app to test that a zipped exe can be unzipped and run on different OSes. +Console.WriteLine("Hello, World!"); \ No newline at end of file diff --git a/test/ZippedExe/ZippedExe.csproj b/test/ZippedExe/ZippedExe.csproj new file mode 100644 index 000000000..1e1505db3 --- /dev/null +++ b/test/ZippedExe/ZippedExe.csproj @@ -0,0 +1,10 @@ + + + + Exe + net8.0 + enable + enable + + +