Added UTF-8 detection for headers

Author: jstedfast

MimeKit/AsyncMimeReader.cs

@@ -131,7 +131,6 @@ async Task StepMboxMarkerAsync (CancellationToken cancellationToken)
 
 		async Task StepHeadersAsync (CancellationToken cancellationToken)
 		{
-			var options = DetectMimeComplianceViolations ? ByteDetectionOptions.Detect8Bit | ByteDetectionOptions.DetectNulls : ByteDetectionOptions.None;
 			int headersBeginLineNumber = lineNumber;
 			var eof = false;
 
@@ -155,6 +154,7 @@ async Task StepHeadersAsync (CancellationToken cancellationToken)
 			await ReadAheadAsync (ReadAheadSize, 0, cancellationToken).ConfigureAwait (false);
 
 			do {
+				var options = DetectMimeComplianceViolations ? ByteDetectionOptions.Detect8Bit | ByteDetectionOptions.DetectNulls : ByteDetectionOptions.None;
 				var beginOffset = GetOffset (inputIndex);
 				var beginLineNumber = lineNumber;
 				int left = inputEnd - inputIndex;
@@ -282,12 +282,13 @@ async Task StepHeadersAsync (CancellationToken cancellationToken)
 				}
 
 				bool midline = true;
+				bool ascii = true;
 
 				// Consume the header value.
 				do {
 					unsafe {
 						fixed (byte* inbuf = input) {
-							if (StepHeaderValue (inbuf, ref options, ref midline))
+							if (StepHeaderValue (inbuf, ref options, ref midline, ref ascii))
 								break;
 						}
 					}
@@ -310,7 +311,7 @@ async Task StepHeadersAsync (CancellationToken cancellationToken)
 					return;
 				}
 
-				var header = CreateHeader (beginOffset, beginLineNumber, fieldNameLength, headerFieldLength, invalid);
+				var header = CreateHeader (beginOffset, beginLineNumber, fieldNameLength, headerFieldLength, invalid, ascii);
 
 				await OnHeaderReadAsync (header, beginLineNumber, cancellationToken).ConfigureAwait (false);
 			} while (!eof);

MimeKit/MimeComplianceViolation.cs

@@ -182,7 +182,7 @@ public enum MimeComplianceViolation
 		/// US-ASCII should be encoded using the encoding mechanism described in the MIME specification and/or should be
 		/// valid UTF-8 as allowed in the Internationalized Email Headers specification.
 		/// </remarks>
-		Unexpected8BitBytesInHeaders,
+		Unexpected8BitBytesInHeader,
 
 		/// <summary>
 		/// A MIME part's body contained 8-bit content where only 7-bit content was expected.

MimeKit/MimeKit.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <Description>An Open Source library for creating and parsing MIME, S/MIME, PGP messages on desktop and mobile platforms.</Description>

MimeKit/MimeReader.cs

@@ -31,6 +31,7 @@
 #endif
 using System.Threading;
 using System.Diagnostics;
+using System.Text.Unicode;
 using System.Threading.Tasks;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
@@ -1777,7 +1778,7 @@ void StepHeaderField (int headerFieldLength)
 			inputIndex += headerIndex;
 		}
 
-		unsafe bool StepHeaderValue (byte* inbuf, ref ByteDetectionOptions options, ref bool midline)
+		unsafe bool StepHeaderValue (byte* inbuf, ref ByteDetectionOptions options, ref bool midline, ref bool ascii)
 		{
 			byte* start = inbuf + inputIndex;
 			byte* inend = inbuf + inputEnd;
@@ -1792,8 +1793,11 @@ unsafe bool StepHeaderValue (byte* inbuf, ref ByteDetectionOptions options, ref
 					inptr = ParseUtils.EndOfLine (inptr, inend + 1, options, out var detected);
 
 					if ((detected & ByteDetectionResults.Detected8Bit) != 0) {
-						OnMimeComplianceViolation (MimeComplianceViolation.Unexpected8BitBytesInHeaders, lineBeginOffset, lineNumber);
+						// Note: we don't emit Unexpected8BitBytesInHeader here because 8-bit might only indicate UTF-8 which is valid in headers
+						// according to RFC 6532. Instead, we'll just track that this header value contains non-ASCII text and check for valid UTF-8
+						// once we've got the full value.
 						options &= ~ByteDetectionOptions.Detect8Bit;
+						ascii = false;
 					}
 
 					if ((detected & ByteDetectionResults.DetectedNulls) != 0) {
@@ -1922,7 +1926,7 @@ unsafe bool TryCheckMboxMarkerWithinHeaderBlock (byte* inbuf)
 			return true;
 		}
 
-		Header CreateHeader (long beginOffset, int beginLineNumber, int fieldNameLength, int headerFieldLength, bool invalid)
+		Header CreateHeader (long beginOffset, int beginLineNumber, int fieldNameLength, int headerFieldLength, bool invalid, bool ascii)
 		{
 			byte[] field, value;
 
@@ -1942,6 +1946,23 @@ Header CreateHeader (long beginOffset, int beginLineNumber, int fieldNameLength,
 				Offset = beginOffset
 			};
 
+			if (DetectMimeComplianceViolations) {
+				if (invalid) {
+					// This means that the field name itself contains all of the data and is invalid. Check for null bytes *and* non-UTF-8 text.
+					var fieldSpan = field.AsSpan ();
+					int index = fieldSpan.IndexOf ((byte) '\0');
+
+					if (index != -1)
+						OnMimeComplianceViolation (MimeComplianceViolation.UnexpectedNullBytesInHeader, beginOffset + index, beginLineNumber);
+
+					if (!Utf8.IsValid (fieldSpan))
+						OnMimeComplianceViolation (MimeComplianceViolation.Unexpected8BitBytesInHeader, beginOffset, beginLineNumber);
+				} else if (!ascii) {
+					if (!Utf8.IsValid (value))
+						OnMimeComplianceViolation (MimeComplianceViolation.Unexpected8BitBytesInHeader, beginOffset, beginLineNumber);
+				}
+			}
+
 			UpdateHeaderState (header, beginOffset, beginLineNumber);
 			headerCount++;
 
@@ -1950,7 +1971,6 @@ Header CreateHeader (long beginOffset, int beginLineNumber, int fieldNameLength,
 
 		unsafe void StepHeaders (byte* inbuf, CancellationToken cancellationToken)
 		{
-			var options = DetectMimeComplianceViolations ? ByteDetectionOptions.Detect8Bit | ByteDetectionOptions.DetectNulls : ByteDetectionOptions.None;
 			int headersBeginLineNumber = lineNumber;
 			var eof = false;
 
@@ -1974,6 +1994,7 @@ unsafe void StepHeaders (byte* inbuf, CancellationToken cancellationToken)
 			ReadAhead (ReadAheadSize, 0, cancellationToken);
 
 			do {
+				var options = DetectMimeComplianceViolations ? ByteDetectionOptions.Detect8Bit | ByteDetectionOptions.DetectNulls : ByteDetectionOptions.None;
 				var beginOffset = GetOffset (inputIndex);
 				var beginLineNumber = lineNumber;
 				int left = inputEnd - inputIndex;
@@ -2080,9 +2101,10 @@ unsafe void StepHeaders (byte* inbuf, CancellationToken cancellationToken)
 				}
 
 				bool midline = true;
+				bool ascii = true;
 
 				// Consume the header value.
-				while (!StepHeaderValue (inbuf, ref options, ref midline)) {
+				while (!StepHeaderValue (inbuf, ref options, ref midline, ref ascii)) {
 					if (ReadAhead (1, 0, cancellationToken) == 0) {
 						if (DetectMimeComplianceViolations) {
 							if (midline)
@@ -2101,7 +2123,7 @@ unsafe void StepHeaders (byte* inbuf, CancellationToken cancellationToken)
 					return;
 				}
 
-				var header = CreateHeader (beginOffset, beginLineNumber, fieldNameLength, headerFieldLength, invalid);
+				var header = CreateHeader (beginOffset, beginLineNumber, fieldNameLength, headerFieldLength, invalid, ascii);
 
 				OnHeaderRead (header, beginLineNumber, cancellationToken);
 			} while (!eof);

MimeKit/Utils/Utf8.cs

@@ -0,0 +1,108 @@
+//
+// Utf8.cs
+//
+// Author: Jeffrey Stedfast <jestedfa@microsoft.com>
+//
+// Copyright (c) 2013-2026 .NET Foundation and Contributors
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+//
+
+#if NETFRAMEWORK || NETSTANDARD || !NET8_0_OR_GREATER
+
+using System;
+using System.Runtime.CompilerServices;
+
+using Microsoft.Win32;
+
+namespace System.Text.Unicode
+{
+	static class Utf8
+	{
+		/// <summary>
+		/// Validate that the value is well-formed UTF-8.
+		/// </summary>
+		/// <remarks>
+		/// Validates that the value is well-formed UTF-8.
+		/// </remarks>
+		/// <param name="value">The byte string.</param>
+		/// <returns><see langword="true"/> if the value is well-formed UTF-8; otherwise, <see langword="false"/>.</returns>
+		public static bool IsValid (ReadOnlySpan<byte> value)
+		{
+			int index = 0;
+
+			if (value.IsEmpty)
+				return true;
+
+			while (index < value.Length) {
+				uint u = ReadUnichar (value, ref index);
+
+				if (u == 0xfffe)
+					return false;
+			}
+
+			return true;
+		}
+
+		static uint ReadUnichar (ReadOnlySpan<byte> value, ref int index)
+		{
+			uint u = 0;
+			byte c, r;
+
+			if (index >= value.Length)
+				return 0;
+
+			r = value[index++];
+
+			if (r < 0x80) {
+				// simple ascii character
+				u = r;
+			} else if (r < 0xfe) {
+				// mask for utf-8 length bits
+				uint mask = 0x7f80;
+
+				u = r;
+
+				do {
+					if (index >= value.Length)
+						return 0xfffe;
+
+					c = value[index++];
+					if ((c & 0xc0) != 0x80) {
+						// invalid utf-8 sequence
+						return 0xfffe;
+					}
+					
+					u = (u << 6) | ((uint) c & 0x3f);
+					r <<= 1;
+					mask <<= 5;
+				} while ((r & 0x40) != 0);
+
+				u &= ~mask;
+			} else {
+				// invalid utf-8 start character
+				return 0xfffe;
+			}
+
+			return u;
+		}
+	}
+}
+
+#endif