moved away from editing dockerfiles with set_versions to using Docker ARG flags and a versions.ps1 user file. Allow switching the base docker image.

Author: mitchcapper

.dockerignore

@@ -0,0 +1,2 @@
+versions.ps1
+.dockerignore

.gitignore

@@ -4,24 +4,24 @@ RUN $env:Path = 'c:/Program Files/Git/bin/;' + $env:Path;setx /M PATH $env:Path;
 RUN Invoke-WebRequest 'https://storage.googleapis.com/chrome-infra/depot_tools.zip' -OutFile 'C:/code/depot_tools.zip';
 RUN Expand-Archive C:/code/depot_tools.zip -DestinationPath "C:/code/depot_tools";
 RUN $env:Path = 'c:/code/depot_tools/;' + $env:Path;setx /M PATH $env:Path;
-ENV GN_DEFINES is_debug=false
+ARG GN_DEFINES="is_debug=false"
 ENV DEPOT_TOOLS_WIN_TOOLCHAIN 0
 ENV CEF_USE_GN 1
 ENV GYP_MSVS_VERSION 2017
-ENV DUAL_BUILD 0
-ENV GYP_DEFINES target_arch=x64 
-RUN setX /M DEPOT_TOOLS_WIN_TOOLCHAIN 0;setX /M GYP_MSVS_VERSION 2017;setx /M GN_DEFINES "$env:GN_DEFINES";setx /M GYP_DEFINES "$env:GYP_DEFINES";
-ENV CHROME_BRANCH 3239
+ARG DUAL_BUILD="0"
+ARG GYP_DEFINES="target_arch=x64 "
+RUN setX /M DEPOT_TOOLS_WIN_TOOLCHAIN 0;setX /M GYP_MSVS_VERSION 2017;setx /M GN_DEFINES "$env:GN_DEFINES";setx /M GYP_DEFINES "$env:GYP_DEFINES";setx /M DUAL_BUILD "$env:DUAL_BUILD";
+ARG CHROME_BRANCH=3239
 ADD https://bitbucket.org/chromiumembedded/cef/raw/master/tools/automate/automate-git.py c:/code/automate/
 RUN cd c:/code/depot_tools/;update_depot_tools.bat;git checkout .\cipd.ps1;update_depot_tools.bat;
 
 # Comment out the next step if you want to do it by hand
-ADD cef_build.ps1 c:/code
+ADD functions.ps1 cef_build.ps1 c:/code/
 RUN c:/code/cef_build.ps1
 
 
 
 
-
-
-
+
+
+

Dockerfile_cef

@@ -1,3 +1,4 @@
 FROM vs
+SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
 ADD binaries.zip c:/code/
 RUN Expand-Archive c:\code\binaries.zip  -DestinationPath "C:\code\binaries";

Dockerfile_cef_compiled

@@ -3,14 +3,14 @@ RUN MKDIR c:/code/cefsharp
 WORKDIR c:/code/cefsharp
 RUN $env:Path = 'c:/Program Files/Git/bin/;' + $env:Path;setx /M PATH $env:Path;
 RUN git clone https://github.com/cefsharp/CefSharp.git .
-ENV CEFSHARP_BRANCH cefsharp/63
+ARG CEFSHARP_BRANCH cefsharp/63
 #This line and the one two below are due to waiting to use the new vs2017 compat build script
 RUN cp build.ps1 ../
 RUN git checkout $env:CEFSHARP_BRANCH;
 RUN cp ../build.ps1 .
-ENV CEFSHARP_VERSION 63.0.90
-ENV CEF_VERSION_STR auto
-ENV CHROME_BRANCH 3239
+ARG CEFSHARP_VERSION 63.0.90
+ARG CEF_VERSION_STR auto
+ARG CHROME_BRANCH 3239
 ENV PACKAGE_SOURCE C:/code/cef-binary/cef-binary-master/NuGet
 ADD cefsharp_set_versions_and_restore.ps1 c:/code/cefsharp
 RUN ./cefsharp_set_versions_and_restore.ps1
@@ -19,4 +19,4 @@ RUN ./cefsharp_set_versions_and_restore.ps1
 ENV VS141COMNTOOLS C:/Program Files (x86)/Microsoft Visual Studio/2017/Community/VC/Auxiliary/Build
 RUN [Net.ServicePointManager]::SecurityProtocol=[Net.SecurityProtocolType]::Tls12;./build.ps1 -Target vs2017 -Version $env:CEFSHARP_VERSION -AssemblyVersion $env:CEFSHARP_VERSION; 
 RUN cp $env:PACKAGE_SOURCE/*.nupkg C:/code/cefsharp/nuget/
-RUN Compress-Archive -Path C:/code/cefsharp/nuget/*.nupkg -CompressionLevel Fastest -DestinationPath C:\packages_cefsharp
+RUN Compress-Archive -Path C:/code/cefsharp/nuget/*.nupkg -CompressionLevel Fastest -DestinationPath C:\packages_cefsharp

Dockerfile_cefsharp

@@ -1,4 +1,5 @@
-FROM microsoft/dotnet-framework:4.7.1-windowsservercore
+ARGS BASE_DOCKER_FILE="microsoft/dotnet-framework:4.7.1-windowsservercore-1709"
+FROM $BASE_DOCKER_FILE
 SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
 
 RUN mkdir c:\code\automate;mkdir c:\temp\;mkdir c:\code\depot_tools;

Dockerfile_vs

@@ -79,11 +79,11 @@ With the Azure F32 v2 host above the total estimated build time is about 3.5 hou
 
 
 ## Build Process
-Once docker is setup and running copy this repo to a local folder.  Edit set_version.ps1 and change the version strings to match what you want then run it to have it update the dockerfiles.  
+Once docker is setup and running copy this repo to a local folder.  Copy versions_src.ps1 to versions.ps1 and change the version strings to match what you want.  NOTE BASE_DOCKER_FILE must match the same kernel as the host machine IF you are using process isolation mode.  This means you cannot use the 1709 image on an older host and you can use and older image on a 1709 host.  Either base file is fine however to use just match it to the host.  
 
-Next run build.ps1 and if you are lucky you will end up with a cefsharp_packages.zip file with all the nupkg files you need:)  Beware that as docker can be flaky you may need to call build.ps1 a few times.  It should largely just resume.
+Next run build.ps1 and if you are lucky you will end up with a cefsharp_packages.zip file with all the nupkg files you need:)  Beware that as docker might be flaky you may need to call build.ps1 a few times.  It should largely just resume.
 
-To be safer you can run each build command by hand.  The hardest (longest) build step if the CEF build at the start. You can comment out the last step in the dockerfile and manually do that step and commit it. To do so comment out the final build step in Dockerfile_cef then run the following:
+To be safer you can run the biggest build command by hand. The hardest (longest) build step if the CEF build at the start. You can comment out the last step in the dockerfile and manually do that step and commit it.  Infact you can just docker run the build image from before than manually call cef_build.ps1 one or more times (it should do a decent job at auto-resuming) until success.  If you are using a proper host with enough ram it should be able to automatically  build 9 times out of 10 (if not higher) with its current redundant tries.  Of course if you prefer to manually run the commands from it you can do that too. To do so comment out the final build step in Dockerfile_cef then run the following:
 ```
 	#So if the autmate-git.py doesn't work (if something errors out it doesn't always stop at the right point) try running the build steps manually that are there.
 	# From the c:/code/chromium/src folder run the build hooks to download tools: gclient runhooks
@@ -94,8 +94,7 @@ To be safer you can run each build command by hand.  The hardest (longest) build
 	# cd C:/code/chromium/src/cef/tools/
 	# C:/code/chromium/src/cef/tools/make_distrib.bat --ninja-build --allow-partial;
 	# c:/code/chromium/src/cef/tools/make_distrib.bat --ninja-build --allow-partial --x64-build;
-
-	# Allow partial needed if not building debug builds
+	# Allow partial needed if not building debug builds, make sure to run it when done or run the cef_build last few commands to create the archive with the result and to clean up the workspace of the source files.
 ```
 ### Dual Build Flag
 Note the DUAL_BUILD may speed up builds by running x86 and x64 builds concurrently (each with 1/2 as many threads).  This is primarily useful during linking.  Linking is largely single threaded and takes awhile and is single thread CPU bound (given enough IO). The main issue is memory usage.  If both linking steps run at once you may need nearly 50GB of memory at once (in worst case).  It would be better if they linked at slightly separate times but as every compute system was different there did not seem to be a good way to determine how long to sleep for to make it most efficient.
@@ -109,7 +108,7 @@ Note the DUAL_BUILD may speed up builds by running x86 and x64 builds concurrent
 - Sometimes docker may start to mis-behave often restarting docker may fix the problem.  Sometimes a full reboot is needed.
 
 ### How requirements were determined
-- Space: windows base ~10 gigs, ~6 gigs for the finished visual studio build image.  Another 20 or so when done with cefsharp.   Chrome will take 200 gigs or so during build for the VHD, we remove the bulk of this before it finishes though.  So for docker storage I would recommend 16 + 200 = ~ 220 gigs of space + some buffer so maybe 250GB. 
+- Space: windows base ~6 gigs, ~9 gigs for the finished visual studio build image.  Another 20 or so when done with cefsharp.   Chrome will take 200 gigs or so during build for the VHD, we remove the bulk of this before it finishes though.  So for docker storage I would recommend 16 + 200 = ~ 220 gigs of space + some buffer so maybe 250GB. 
 - Memory: For Chrome 63 bare minimum memory requirements (actual + page file) for JUST the linker is x86: 24.2 GB x64: 25.7 GB. I would make sure you have atleast 32 gigs of ram to be safe with OS and other overhead.
 
 

README.md

@@ -1,78 +1,52 @@
 [CmdletBinding()]
 Param(
-	[Switch] $NoSkip
+	[Switch] $NoSkip,
+	[Switch] $NoMemoryWarn
 )
-
+$WorkingDir = split-path -parent $MyInvocation.MyCommand.Definition;
+. (Join-Path $WorkingDir 'functions.ps1')
+#Always read the source file first incase of a new variable.
+. (Join-Path $WorkingDir "versions_src.ps1")
+#user overrides
+if (Test-Path ./versions.ps1 -PathType Leaf){
+	. (Join-Path $WorkingDir "versions.ps1")
+}
 Set-StrictMode -version latest;
 $ErrorActionPreference = "Stop";
 if ((Get-MpPreference).DisableRealtimeMonitoring -eq $false){
 	Write-Host Warning, windows defender is enabled it will slow things down. -Foreground Red 
 }
-
-$last_time = Get-Date;
-function out {
-	Param(
-		[Parameter(Mandatory = $true, ParameterSetName = "ByPath", Position = 0)]
-  		$FilePath,
-		[Parameter(Mandatory = $true,ValueFromPipeline=$true,ValueFromPipelineByPropertyName=$true, ParameterSetName = "ByPath", Position = 1)]
-  		$InputObject,
-  		[switch] $Append
-  		)
-		Begin {
-			if (! $Append){
-				Out-File -FilePath $FilePath -Encoding ASCII;
-			}
-		}
-		Process {
-				Out-File -Append -InputObject $InputObject -FilePath $FilePath	-Encoding ASCII
+if (! $NoMemoryWarn){
+	$page_files = Get-CimInstance Win32_PageFileSetting;
+	$os = Get-Ciminstance Win32_OperatingSystem;
+	$min_gigs = 27;
+	$warning = "linking may take around $min_gigs during linking";
+	if ($VAR_DUAL_BUILD -eq "1"){
+		$warning="dual build mode is enabled and may use 50+ GB if both releases link at once.";
+		$min_gigs = 50;
+	}
+	if (($os.FreePhysicalMemory/1mb + $os.FreeSpaceInPagingFiles/1mb) -lt $min_gigs) { #if the memory isn't yet avail with the page files and they have a set size lets try to compute it that way
+		$total_memory_gb = $os.FreePhysicalMemory/1mb;
+		foreach ($file in $page_files){
+			$total_memory_gb += $file.MaximumSize/1kb; #is zero if system managed, then we really don't know how big it could be.
 		}
-}
-
-Function RunProc{
-    [CmdletBinding()]
-    Param($proc,$opts,
-    [switch] $errok)
-    Write-Verbose "Running: $proc $opts"
-    if ($proc.ToUpper().EndsWith(".PL")){
-        $path = (C:\Windows\system32\where.exe $proc | Out-String).Trim()
-        $opts = "$path $opts";
-        $proc="perl";       
-    }
-    $pinfo = New-Object System.Diagnostics.ProcessStartInfo
-    $pinfo.FileName = $proc
-    $working = Convert-Path .
-    $pinfo.WorkingDirectory = $working
-    $pinfo.UseShellExecute = $false
-    $pinfo.Arguments = $opts
-    $p = New-Object System.Diagnostics.Process
-    $p.StartInfo = $pinfo
-    $p.Start() | Out-Null
-    $p.WaitForExit()
-    if ($p.ExitCode -ne 0 -and -not $errok){
-        throw "Process $proc $opts exited with non zero code aborting!" 
-    }
-    if ($errok){
-        return $p.ExitCode;
-    }
-}
+		if ($total_memory_gb -lt $min_gigs){
+			if (! (confirm("Warning $warning.  Your machine may not have enough memory, make sure your page files can grow to allow it. (Disable this warning with -$NoMemoryWarn flag). Do you want to proceed?"))){
+				exit 1;
+			}
 
-Function TimerNow($name){
-	$now = Get-Date;
-	$diff = ($now -  $start_time).TotalSeconds.ToString("0.0");
-	Write-Host $name took $diff  -ForegroundColor Green;
-	$last_time = $now;
+		}
+	}
 }
-$start_time = Get-Date;
-$end_time = Get-Date;
 
 
 echo packages_cefsharp.zip | out .dockerignore
 echo binaries.zip | out .dockerignore -Append
 if (! (Test-Path ./binaries.zip -PathType Leaf) -or $NoSkip){
 	TimerNow("Starting");
-	RunProc -proc "docker" -opts "build -f Dockerfile_vs -t vs ."
+	RunProc -proc "docker" -opts "build --build-arg BASE_DOCKER_FILE=`"$VAR_BASE_DOCKER_FILE`" -f Dockerfile_vs -t vs ."
 	TimerNow("VSBuild");
-	RunProc -proc "docker" -opts "build -f Dockerfile_cef -t cef ."
+	RunProc -proc "docker" -opts "build --build-arg DUAL_BUILD=`"$VAR_DUAL_BUILD`" --build-arg GN_DEFINES=`"$VAR_GN_DEFINES`" --build-arg GYP_DEFINES=`"$VAR_GYP_DEFINES`" --build-arg CHROME_BRANCH=`"$VAR_CHROME_BRANCH`" -f Dockerfile_cef -t cef ."
 	TimerNow("CEF Build");
 	docker rm cef
 	RunProc -proc "docker" -opts "run --name cef cef cmd /C echo CopyVer"
@@ -87,7 +61,7 @@ if (! (Test-Path ./binaries.zip -PathType Leaf) -or $NoSkip){
 }
 RunProc -proc "docker" -opts "build -f Dockerfile_cef_binary -t cef_binary ."
 TimerNow("CEF Binary compile");
-RunProc -proc "docker" -opts "build -f Dockerfile_cefsharp -t cefsharp ."
+RunProc -proc "docker" -opts "build --build-arg CEFSHARP_BRANCH=`"$VAR_CEFSHARP_BRANCH`" --build-arg CEFSHARP_VERSION=`"$VAR_CEFSHARP_VERSION`" --build-arg CEF_VERSION_STR=`"$VAR_CEF_VERSION_STR`" --build-arg CHROME_BRANCH=`"$VAR_CHROME_BRANCH`" -f Dockerfile_cefsharp -t cefsharp ."
 TimerNow("CEFSharp compile");
 docker rm cefsharp
 RunProc -proc "docker" -opts "run --name cefsharp cefsharp cmd /C echo CopyVer"

build.ps1

@@ -1,37 +1,8 @@
 Set-StrictMode -version latest;
 $ErrorActionPreference = "Stop";
-Function RunProc{
-    [CmdletBinding()]
-    Param($proc,$opts,
-    [switch] $errok,
-    [switch] $no_wait
-    )
-    if ($proc.ToUpper().EndsWith(".PL")){
-        $path = (C:\Windows\system32\where.exe $proc | Out-String).Trim()
-        $opts = "$path $opts";
-        $proc="perl";       
-    }
-    $pinfo = New-Object System.Diagnostics.ProcessStartInfo
-    $pinfo.FileName = $proc
-    $working = Convert-Path .
-    Write-Host "Running: $proc $opts in $working"  -Foreground Green    
-    $pinfo.WorkingDirectory = $working
-    $pinfo.UseShellExecute = $false
-    $pinfo.Arguments = $opts
-    $p = New-Object System.Diagnostics.Process
-    $p.StartInfo = $pinfo
-    $p.Start() | Out-Null
-    if ($no_wait){
-    	return $p;
-    }
-    $p.WaitForExit()
-    if ($p.ExitCode -ne 0 -and -not $errok){
-        throw "Process $proc $opts exited with non zero code: $($p.ExitCode) aborting!" 
-    }
-    if ($errok){
-        return $p.ExitCode;
-    }
-}
+$WorkingDir = split-path -parent $MyInvocation.MyCommand.Definition;
+. (Join-Path $WorkingDir 'functions.ps1')
+
 $build_args_add = "";
 if ($env:DUAL_BUILD -eq "1"){
 	$cores = (Get-WmiObject -class Win32_processor).NumberOfLogicalProcessors + 2; #ninja defaults to number of procs + 2 
@@ -43,7 +14,7 @@ if ($env:DUAL_BUILD -eq "1"){
 Function RunBuild{
     [CmdletBinding()]
     Param($build_args_add,$version)
-    return RunProc -proc "c:/code/depot_tools/ninja.exe" -opts "$build_args_add -C out/Release_GN_$version cefclient" -no_wait;
+    return RunProc "host" -proc "c:/code/depot_tools/ninja.exe" -opts "$build_args_add -C out/Release_GN_$version cefclient" -no_wait;
 }
 RunProc -proc "c:/code/depot_tools/python.bat" -errok -opts "c:/code/automate/automate-git.py --download-dir=c:/code --branch=$env:CHROME_BRANCH --no-build --no-debug-build --no-distrib";
 Set-Location -Path c:/code/chromium/src/cef;

cef_build.ps1

@@ -0,0 +1,68 @@
+Set-StrictMode -version latest;
+$ErrorActionPreference = "Stop";
+Function RunProc{
+    [CmdletBinding()]
+    Param($proc,$opts,
+    [switch] $errok,
+    [switch] $no_wait,
+    [ValidateSet("verbose","host","none")] 
+	[String] $verbose_mode="host"
+    )
+    if ($proc.ToUpper().EndsWith(".PL")){
+        $path = (C:\Windows\system32\where.exe $proc | Out-String).Trim()
+        $opts = "$path $opts";
+        $proc="perl";       
+    }
+    $pinfo = New-Object System.Diagnostics.ProcessStartInfo;
+    $pinfo.FileName = $proc;
+    $working = Convert-Path .;
+    $verbose_str = "Running: $proc $opts in $working";
+    if ($verbose_mode -eq "host"){
+    	Write-Host $verbose_str -Foreground Green;
+    }elseif($verbose_mode -eq "verbose"){
+		Write-Verbose $verbose_str;
+    }
+    $pinfo.WorkingDirectory = $working;
+    $pinfo.UseShellExecute = $false;
+    $pinfo.Arguments = $opts;
+    $p = New-Object System.Diagnostics.Process;
+    $p.StartInfo = $pinfo;
+    $p.Start() | Out-Null;
+    if ($no_wait){
+    	return $p;
+    }
+    $p.WaitForExit();
+    if ($p.ExitCode -ne 0 -and -not $errok){
+        throw "Process $proc $opts exited with non zero code: $($p.ExitCode) aborting!" ;
+    }
+    if ($errok){
+        return $p.ExitCode;
+    }
+}
+function out {
+	Param(
+		[Parameter(Mandatory = $true, ParameterSetName = "ByPath", Position = 0)]
+  		$FilePath,
+		[Parameter(Mandatory = $true,ValueFromPipeline=$true,ValueFromPipelineByPropertyName=$true, ParameterSetName = "ByPath", Position = 1)]
+  		$InputObject,
+  		[switch] $Append
+  		)
+		Begin {
+			if (! $Append){
+				Out-File -FilePath $FilePath -Encoding ASCII;
+			}
+		}
+		Process {
+				Out-File -Append -InputObject $InputObject -FilePath $FilePath	-Encoding ASCII
+		}
+}
+Function confirm($str){
+	return $PSCmdlet.ShouldContinue($str, "");
+}
+$last_time = Get-Date;
+Function TimerNow($name){
+	$now = Get-Date;
+	$diff = ($now -  $last_time).TotalSeconds.ToString("0.0");
+	Write-Host $name took $diff  -ForegroundColor Green;
+	$last_time = $now;
+}