Enumerable as content, some other things

Author: SchlenkR

.vscode/launch.json

@@ -1,41 +1,34 @@
 {
-    "version": "2.0.0",
-    "tasks": [
-        {
-            "label": "build",
-            "command": "dotnet",
-            "type": "process",
-            "args": [
-                "build",
-                "${workspaceFolder}/src/Test.CSharp/Test.CSharp.csproj",
-                "/property:GenerateFullPaths=true",
-                "/consoleloggerparameters:NoSummary"
-            ],
-            "problemMatcher": "$msCompile"
-        },
-        {
-            "label": "publish",
-            "command": "dotnet",
-            "type": "process",
-            "args": [
-                "publish",
-                "${workspaceFolder}/src/Test.CSharp/Test.CSharp.csproj",
-                "/property:GenerateFullPaths=true",
-                "/consoleloggerparameters:NoSummary"
-            ],
-            "problemMatcher": "$msCompile"
-        },
-        {
-            "label": "watch",
-            "command": "dotnet",
-            "type": "process",
-            "args": [
-                "watch",
-                "run",
-                "--project",
-                "${workspaceFolder}/src/Test.CSharp/Test.CSharp.csproj"
-            ],
-            "problemMatcher": "$msCompile"
-        }
-    ]
-}
+  "version": "2.0.0",
+  "options": {
+    "cwd": "."
+  },
+  "windows": {
+    "options": {
+      "shell": {
+        "executable": "C:\\Program Files\\Git\\bin\\bash.exe",
+        "args": ["-c"]
+      }
+    }
+  },
+  "tasks": [
+    {
+      "label": "FsHttp: Build Solution",
+      "type": "shell",
+      "command": "./build.sh",
+      "group": {
+        "kind": "build",
+        "isDefault": false
+      }
+    },
+    {
+      "label": "FsHttp: Test Solution",
+      "type": "shell",
+      "command": "./test.sh",
+      "group": {
+        "kind": "build",
+        "isDefault": false
+      }
+    }
+  ]
+}

.vscode/tasks.json

@@ -16,6 +16,10 @@
     <FsDocsReleaseNotesLink>https://www.nuget.org/packages/FsHttp#release-body-tab</FsDocsReleaseNotesLink>
 
     <PackageReleaseNotes>
+      15.0.0
+      - Added 'enumerable' and 'enumerablePart' as body content functions
+      - Removed Utf8StringBufferingStream
+
       14.5.1
       - Fixed untracked bug: using config_useBaseUrl as http template won't crash when printing unfinished requests
 

Directory.Build.props

@@ -0,0 +1 @@
+dotnet build

build.sh

@@ -16,7 +16,7 @@ if [ -d ./.fsdocs ]; then
 fi
 
 dotnet tool restore
-dotnet build ./src/FsHttp/FsHttp.fsproj -c Release -f net6.0
+dotnet build ./src/FsHttp/FsHttp.fsproj -c Release -f net8.0
 
 # what a hack...
 if [ -z "$1" ]; then
@@ -29,7 +29,7 @@ dotnet fsdocs \
     $mode \
     --clean \
     --sourcefolder ./src \
-    --properties Configuration=Release TargetFramework=net6.0 \
+    --properties Configuration=Release TargetFramework=net8.0 \
     --sourcerepo https://github.com/fsprojects/FsHttp/blob/master/src \
     --parameters root /FsHttp/ \
     --output ./.docsOutput \

docu.sh

@@ -0,0 +1,56 @@
+open System
+open System.IO
+open System.Text
+
+let readUtf8StringAsync maxLen (stream: Stream) =
+    async {
+        use reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks = false, bufferSize = 1024, leaveOpen = true)
+        let sb = StringBuilder()
+        let mutable codePointsRead = 0
+        let buffer = Array.zeroCreate<char> 1 // Buffer to read one character at a time
+        while codePointsRead < maxLen && not reader.EndOfStream do
+            // Read the next character asynchronously
+            let! charsRead = reader.ReadAsync(buffer, 0, 1) |> Async.AwaitTask
+            if charsRead > 0 then
+                let c = buffer.[0]
+                sb.Append(c) |> ignore
+                // Check if the character is a high surrogate
+                if Char.IsHighSurrogate(c) && not reader.EndOfStream then
+                    // Read the low surrogate asynchronously and append it
+                    let! nextCharsRead = reader.ReadAsync(buffer, 0, 1) |> Async.AwaitTask
+                    if nextCharsRead > 0 then
+                        let nextC = buffer.[0]
+                        sb.Append(nextC) |> ignore
+                // Increment the code point count
+                codePointsRead <- codePointsRead + 1
+        return sb.ToString()
+    }
+
+let text = "a😉b🙁🙂d"
+
+let test len (expected: string) =
+    let res =
+        new MemoryStream(Encoding.UTF8.GetBytes(text))
+        |> readUtf8StringAsync len
+        |> Async.RunSynchronously
+    let s1 = Encoding.UTF8.GetBytes res |> Array.toList
+    let s2 = Encoding.UTF8.GetBytes expected |> Array.toList
+    let res = (s1 = s2)
+    if not res then
+        printfn ""
+        printfn "count = %d" len
+        printfn "expected = %s" expected
+        printfn ""
+        printfn "Expected: %A" s2
+        printfn ""
+        printfn "Actual  : %A" s1
+        printfn ""
+        printfn " ----------------------------"
+
+test 0 ""
+test 1 "a"
+test 2 "a😉"
+test 3 "a😉b"
+test 4 "a😉b🙁"
+test 5 "a😉b🙁🙂"
+test 6 "a😉b🙁🙂d"

fiddle/utf8StreamStuff.fsx

@@ -329,6 +329,10 @@ type IRequestContext<'self> with
     member this.Stream(context: IRequestContext<BodyContext>, stream) = 
         Body.stream stream context.Self
 
+    [<CustomOperation("enumerable")>]
+    member this.Enumerable(context: IRequestContext<BodyContext>, data) = 
+        Body.enumerable data context.Self
+
     [<CustomOperation("text")>]
     member this.Text(context: IRequestContext<BodyContext>, text) = 
         Body.text text context.Self
@@ -403,6 +407,10 @@ type IRequestContext<'self> with
     member this.StreamPart(context: IRequestContext<#IToMultipartContext>, value, name, ?fileName) =
         Multipart.streamPart value name fileName context.Self
 
+    [<CustomOperation("enumerablePart")>]
+    member this.EnumerablePart(context: IRequestContext<#IToMultipartContext>, value, name, ?fileName) =
+        Multipart.enumerablePart value name fileName context.Self
+
 
 // ---------
 // Config

src/FsHttp/Dsl.CE.fs

@@ -344,6 +344,10 @@ module Body =
             (StreamContent stream)
             context
 
+    let enumerable (sequence: byte seq) (context: IToBodyContext) =
+        let enumerableStream = new EnumerableStream(sequence)
+        stream enumerableStream context
+
     let text (text: string) (context: IToBodyContext) =
         content
             {
@@ -450,6 +454,10 @@ module Multipart =
     let streamPart (value: System.IO.Stream) name fileName (context: IToMultipartContext) =
         part (StreamContent value) name fileName context
 
+    let enumerablePart (value: byte seq) name fileName (context: IToMultipartContext) =
+        let enumerableStream = new EnumerableStream(value)
+        streamPart enumerableStream name fileName context
+
 
 module Config =
 

src/FsHttp/Dsl.fs

@@ -64,87 +64,36 @@ module Url =
         let b = (norm url2).TrimStart(delTrim).TrimEnd(delTrim)
         a + sdel + b
 
-// TODO: Inefficient
-type Utf8StringBufferingStream(baseStream: Stream, readBufferLimit: int option) =
-    inherit Stream()
-    let notSeekable () = failwith "Stream is not seekable."
-    let notWritable () = failwith "Stream is not writable."
-    let readBuffer = ResizeArray<byte>()
-    override _.Flush() = baseStream.Flush()
-
-    override _.Read(buffer, offset, count) =
-        let readCount = baseStream.Read(buffer, offset, count)
-
-        match readCount, readBufferLimit with
-        | 0, _ -> ()
-        | readCount, None -> readBuffer.AddRange(buffer[offset .. readCount - 1])
-        | readCount, Some limit ->
-            let remaining = limit - readBuffer.Count
-            let copyCount = min remaining readCount
-
-            if copyCount > 0 then
-                readBuffer.AddRange(buffer[offset .. copyCount - 1])
-
-        readCount
-
-    override _.Seek(_, _) = notSeekable ()
-    override _.SetLength(_) = notWritable ()
-    override _.Write(_, _, _) = notWritable ()
-    override _.CanRead = true
-    override _.CanSeek = false
-    override _.CanWrite = false
-    override _.Length = baseStream.Length
-
-    override _.Position
-        with get () = baseStream.Position
-        and set (_) = notSeekable ()
-
-    member _.GetUtf8String() =
-        let buffer = CollectionsMarshal.AsSpan(readBuffer)
-        let s = Encoding.UTF8.GetString(buffer).AsSpan()
-
-        if s.Length = 0 then
-            s.ToString()
-        else
-            let s =
-                if s[s.Length - 1] |> Char.IsHighSurrogate then
-                    s.Slice(0, s.Length - 1)
-                else
-                    s
-
-            s.ToString()
-
 [<RequireQualifiedAccess>]
 module Stream =
-    let readUtf8StringAsync maxUtf16CharCount (stream: Stream) =
-        let sr = new StreamReader(stream, Encoding.UTF8)
-        let sb = StringBuilder()
-        let charBuffer = Array.zeroCreate<char> (50)
-        let mutable lastCharIsHighSurrogate = false
-
-        let rec read () =
-            async {
-                if sb.Length < maxUtf16CharCount then
-                    let! readCount = sr.ReadAsync(charBuffer, 0, charBuffer.Length) |> Async.AwaitTask
-                    let remaining = maxUtf16CharCount - sb.Length
-                    let copyCount = min remaining readCount
-
-                    if copyCount > 0 then
-                        sb.Append(charBuffer, 0, copyCount) |> ignore
-                        lastCharIsHighSurrogate <- charBuffer[copyCount - 1] |> Char.IsHighSurrogate
-                        do! read ()
-            }
-
+    let readUtf8StringAsync maxLen (stream: Stream) =
+        // we could definitely optimize this
         async {
-            do! read ()
-
-            do
-                sr.Dispose()
-                stream.Dispose()
-
-            return sb.ToString(0, (if lastCharIsHighSurrogate then sb.Length - 1 else sb.Length))
+            use reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks = false, bufferSize = 1024, leaveOpen = true)
+            let sb = StringBuilder()
+            let mutable codePointsRead = 0
+            let buffer = Array.zeroCreate<char> 1 // Buffer to read one character at a time
+            while codePointsRead < maxLen && not reader.EndOfStream do
+                // Read the next character asynchronously
+                let! charsRead = reader.ReadAsync(buffer, 0, 1) |> Async.AwaitTask
+                if charsRead > 0 then
+                    let c = buffer.[0]
+                    sb.Append(c) |> ignore
+                    // Check if the character is a high surrogate
+                    if Char.IsHighSurrogate(c) && not reader.EndOfStream then
+                        // Read the low surrogate asynchronously and append it
+                        let! nextCharsRead = reader.ReadAsync(buffer, 0, 1) |> Async.AwaitTask
+                        if nextCharsRead > 0 then
+                            let nextC = buffer.[0]
+                            sb.Append(nextC) |> ignore
+                    // Increment the code point count
+                    codePointsRead <- codePointsRead + 1
+            return sb.ToString()
         }
 
+    let readUtf8StringTAsync maxLength (stream: Stream) =
+        readUtf8StringAsync maxLength stream |> Async.StartAsTask
+
     let copyToCallbackAsync (target: Stream) callback (source: Stream) =
         async {
             let buffer = Array.create 1024 (byte 0)
@@ -218,3 +167,43 @@ module Stream =
         }
 
     let saveFileTAsync fileName source = saveFileAsync fileName source |> Async.StartAsTask
+
+
+type EnumerableStream(source: byte seq) =
+    inherit Stream()
+
+    let enumerator = source.GetEnumerator()
+    let mutable isDisposed = false
+    let mutable position = 0L
+
+    override _.CanRead = true
+    override _.CanSeek = false
+    override _.CanWrite = false
+
+    override _.Length = raise (NotSupportedException())
+
+    override _.Position
+        with get() = position
+        and set(_) = raise (NotSupportedException())
+
+    override _.Flush() = ()
+
+    override _.Read(buffer: byte[], offset: int, count: int) =
+        let bytesToRead = Math.Min(count, buffer.Length - int position)
+        if bytesToRead <= 0 then 0
+        else
+            let mutable bytesRead = 0
+            while bytesRead < bytesToRead && enumerator.MoveNext() do
+                buffer.[offset + bytesRead] <- enumerator.Current
+                bytesRead <- bytesRead + 1
+            position <- position + int64 bytesRead
+            bytesRead
+
+    override _.Seek(_: int64, _: SeekOrigin) = raise (NotSupportedException())
+    override _.SetLength(_: int64) = raise (NotSupportedException())
+    override _.Write(_: byte[], _: int, _: int) = raise (NotSupportedException())
+
+    override _.Dispose(disposing: bool) =
+        if not isDisposed then
+            isDisposed <- true
+            enumerator.Dispose()