Skip to content

Commit 3546fcf

Browse files
CypherPotatoCypherPotato
committed
update benchmarks
1 parent 8a824d6 commit 3546fcf

15 files changed

+150
-65
lines changed

‎README.md

+69-9
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,74 @@
11
# Benchmark test between Sisk and ASP.NET
22

3-
[Host] : .NET 6.0.16 (6.0.1623.17311), X64 RyuJIT AVX2
4-
DefaultJob : .NET 6.0.16 (6.0.1623.17311), X64 RyuJIT AVX2
3+
The tests below were performed to improve the quality of Sisk. All tests are reproducible and were run on a personal machine and not a server. I will soon add tests on a production server.
54

6-
| Method | Mean | Error | StdDev |
7-
|-------------- |---------:|--------:|--------:|
8-
| AspNetMinimal | 101.9 us | 1.13 us | 0.95 us |
9-
| SiskDotNet | 108.4 us | 1.50 us | 1.40 us |
10-
| SiskBFlat | 109.9 us | 1.58 us | 1.48 us |
5+
Both applications were compiled in RELEASE and return a plain "Hello, world!". The K6 version used in these tests was v0.46.0.
116

12-
Both HTTP servers was configured to return an plain `Hello, world!`.
7+
Sisk could handle 44.029 requests per second while ASP.NET got 51.766, using their Kestrel socket implementation. Sisk does uses HttpListener. Sisk is around 15%-20% slower than ASP.NET/Kestrel. We need to improve it.
138

14-
ASP.NET project is using Kestrel.
9+
## Benchmark.NET
10+
11+
```
12+
BenchmarkDotNet v0.13.7, Windows 11 (10.0.22631.2861)
13+
Intel Core i3-10105F CPU 3.70GHz, 1 CPU, 8 logical and 4 physical cores
14+
.NET SDK 8.0.100
15+
[Host] : .NET 8.0.0 (8.0.23.53103), X64 RyuJIT AVX2
16+
DefaultJob : .NET 8.0.0 (8.0.23.53103), X64 RyuJIT AVX2
17+
```
18+
19+
| Framework | Mean | StdDev | StdErr | Op/s |
20+
|-------------- |----------:|---------:|---------:|---------:|
21+
| AspNetMinimal | 96.64 us | 0.380 us | 0.095 us | 10,348.1 |
22+
| SiskDotNet | 110.36 us | 0.383 us | 0.099 us | 9,061.4 |
23+
24+
## K6
25+
26+
Sisk:
27+
28+
```
29+
scenarios: (100.00%) 1 scenario, 500 max VUs, 1m0s max duration (incl. graceful stop):
30+
* default: 500 looping VUs for 30s (gracefulStop: 30s)
31+
32+
33+
data_received..................: 246 MB 8.2 MB/s
34+
data_sent......................: 106 MB 3.5 MB/s
35+
http_req_blocked...............: avg=7.84µs min=0s med=0s max=70.35ms p(90)=0s p(95)=0s
36+
http_req_connecting............: avg=3.88µs min=0s med=0s max=22.57ms p(90)=0s p(95)=0s
37+
http_req_duration..............: avg=11.18ms min=0s med=10.53ms max=111.62ms p(90)=15.34ms p(95)=18.64ms
38+
{ expected_response:true }...: avg=11.18ms min=0s med=10.53ms max=111.62ms p(90)=15.34ms p(95)=18.64ms
39+
http_req_failed................: 0.00% ✓ 0 ✗ 1321362
40+
http_req_receiving.............: avg=41.89µs min=0s med=0s max=93.87ms p(90)=0s p(95)=0s
41+
http_req_sending...............: avg=15.95µs min=0s med=0s max=88.68ms p(90)=0s p(95)=0s
42+
http_req_tls_handshaking.......: avg=0s min=0s med=0s max=0s p(90)=0s p(95)=0s
43+
http_req_waiting...............: avg=11.12ms min=0s med=10.52ms max=81.43ms p(90)=15.27ms p(95)=18.51ms
44+
http_reqs......................: 1321362 44029.579878/s
45+
iteration_duration.............: avg=11.31ms min=0s med=10.58ms max=111.62ms p(90)=15.55ms p(95)=19.48ms
46+
iterations.....................: 1321362 44029.579878/s
47+
vus............................: 500 min=500 max=500
48+
vus_max........................: 500 min=500 max=500
49+
```
50+
51+
ASP.NET Minimal:
52+
53+
```
54+
scenarios: (100.00%) 1 scenario, 500 max VUs, 1m0s max duration (incl. graceful stop):
55+
* default: 500 looping VUs for 30s (gracefulStop: 30s)
56+
57+
58+
data_received..................: 255 MB 8.5 MB/s
59+
data_sent......................: 124 MB 4.1 MB/s
60+
http_req_blocked...............: avg=105.19µs min=0s med=0s max=537.38ms p(90)=0s p(95)=0s
61+
http_req_connecting............: avg=97.47µs min=0s med=0s max=530.34ms p(90)=0s p(95)=0s
62+
http_req_duration..............: avg=9.03ms min=0s med=8.52ms max=306.34ms p(90)=11.24ms p(95)=14.81ms
63+
{ expected_response:true }...: avg=9.03ms min=0s med=8.52ms max=306.34ms p(90)=11.24ms p(95)=14.81ms
64+
http_req_failed................: 0.00% ✓ 0 ✗ 1553207
65+
http_req_receiving.............: avg=48.53µs min=0s med=0s max=296.69ms p(90)=0s p(95)=0s
66+
http_req_sending...............: avg=20.31µs min=0s med=0s max=86.06ms p(90)=0s p(95)=0s
67+
http_req_tls_handshaking.......: avg=0s min=0s med=0s max=0s p(90)=0s p(95)=0s
68+
http_req_waiting...............: avg=8.96ms min=0s med=8.52ms max=196.61ms p(90)=11.14ms p(95)=14.49ms
69+
http_reqs......................: 1553207 51766.383908/s
70+
iteration_duration.............: avg=9.56ms min=0s med=8.52ms max=566.56ms p(90)=11.68ms p(95)=16.93ms
71+
iterations.....................: 1553207 51766.383908/s
72+
vus............................: 500 min=500 max=500
73+
vus_max........................: 500 min=500 max=500
74+
```

‎asp/.config/dotnet-tools.json

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"version": 1,
3+
"isRoot": true,
4+
"tools": {
5+
"dotnet-ef": {
6+
"version": "8.0.0",
7+
"commands": [
8+
"dotnet-ef"
9+
]
10+
}
11+
}
12+
}

‎asp/Program.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,6 @@ public static void Main(string[] args)
99

1010
app.MapGet("/", () => "Hello World!");
1111

12-
app.Run();
12+
app.Run("http://localhost:5000/");
1313
}
1414
}

‎asp/asp.csproj

+6-5
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
<Project Sdk="Microsoft.NET.Sdk.Web">
22

3-
<PropertyGroup>
4-
<TargetFramework>net6.0</TargetFramework>
5-
<Nullable>enable</Nullable>
6-
<ImplicitUsings>enable</ImplicitUsings>
7-
</PropertyGroup>
3+
<PropertyGroup>
4+
<TargetFramework>net8.0</TargetFramework>
5+
<Nullable>enable</Nullable>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<ServerGarbageCollection>true</ServerGarbageCollection>
8+
</PropertyGroup>
89

910
</Project>

‎flat-sisk/.gitignore

-1
This file was deleted.

‎flat-sisk/Program.cs

-14
This file was deleted.

‎flat-sisk/Sisk.Core.dll

-120 KB
Binary file not shown.

‎flat-sisk/build.bat

-1
This file was deleted.

‎flat-sisk/flat-sisk.exe

-4.17 MB
Binary file not shown.

‎sisk-benchmark.sln

+9-3
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,13 @@ Microsoft Visual Studio Solution File, Format Version 12.00
33
# Visual Studio Version 17
44
VisualStudioVersion = 17.6.33723.286
55
MinimumVisualStudioVersion = 10.0.40219.1
6-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "sisk-benchmark", "sisk-benchmark\sisk-benchmark.csproj", "{D75F0003-3E8F-499D-885D-51372C1C53B2}"
6+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "sisk-benchmark", "sisk-benchmark\sisk-benchmark.csproj", "{D75F0003-3E8F-499D-885D-51372C1C53B2}"
77
EndProject
8-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "asp", "asp\asp.csproj", "{2B583AB2-B744-4020-881C-15236C320949}"
8+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "asp", "asp\asp.csproj", "{2B583AB2-B744-4020-881C-15236C320949}"
99
EndProject
10-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "sisk", "sisk\sisk.csproj", "{E2B23FA6-317D-4109-A5DC-001F634A5B17}"
10+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "sisk", "sisk\sisk.csproj", "{E2B23FA6-317D-4109-A5DC-001F634A5B17}"
11+
EndProject
12+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Sisk.Core", "..\core\src\Sisk.Core.csproj", "{FC8A2FA2-8DDF-497C-8CE5-6F7CECEB19FB}"
1113
EndProject
1214
Global
1315
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -27,6 +29,10 @@ Global
2729
{E2B23FA6-317D-4109-A5DC-001F634A5B17}.Debug|Any CPU.Build.0 = Debug|Any CPU
2830
{E2B23FA6-317D-4109-A5DC-001F634A5B17}.Release|Any CPU.ActiveCfg = Release|Any CPU
2931
{E2B23FA6-317D-4109-A5DC-001F634A5B17}.Release|Any CPU.Build.0 = Release|Any CPU
32+
{FC8A2FA2-8DDF-497C-8CE5-6F7CECEB19FB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
33+
{FC8A2FA2-8DDF-497C-8CE5-6F7CECEB19FB}.Debug|Any CPU.Build.0 = Debug|Any CPU
34+
{FC8A2FA2-8DDF-497C-8CE5-6F7CECEB19FB}.Release|Any CPU.ActiveCfg = Release|Any CPU
35+
{FC8A2FA2-8DDF-497C-8CE5-6F7CECEB19FB}.Release|Any CPU.Build.0 = Release|Any CPU
3036
EndGlobalSection
3137
GlobalSection(SolutionProperties) = preSolution
3238
HideSolutionNode = FALSE

‎sisk-benchmark/Program.cs

+23-5
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,38 @@
11
using BenchmarkDotNet.Attributes;
2+
using BenchmarkDotNet.Columns;
3+
using BenchmarkDotNet.Configs;
4+
using BenchmarkDotNet.Diagnosers;
5+
using BenchmarkDotNet.Jobs;
6+
using BenchmarkDotNet.Loggers;
27
using BenchmarkDotNet.Running;
8+
using Microsoft.Extensions.Logging;
39

410
namespace sisk_benchmark;
511

612
public static class Program
713
{
814
private static void Main()
915
{
10-
BenchmarkRunner.Run<SiskVsMinimalBenckmark>();
16+
BenchmarkRunner.Run<Test1>(new Config());
17+
Console.ReadKey();
1118
}
1219
}
1320

14-
public class SiskVsMinimalBenckmark
21+
class Config : ManualConfig
22+
{
23+
public Config()
24+
{
25+
AddLogger(new ConsoleLogger());
26+
AddColumn(new TagColumn("Framework", name => name));
27+
AddColumn(StatisticColumn.Mean);
28+
AddColumn(StatisticColumn.StdDev);
29+
AddColumn(StatisticColumn.StdErr);
30+
AddColumn(StatisticColumn.OperationsPerSecond);
31+
AddDiagnoser(MemoryDiagnoser.Default);
32+
}
33+
}
34+
35+
public class Test1
1536
{
1637
private HttpClient? _httpClient;
1738

@@ -23,7 +44,4 @@ public class SiskVsMinimalBenckmark
2344

2445
[Benchmark]
2546
public async Task SiskDotNet() => await _httpClient!.GetStringAsync("http://localhost:5222/");
26-
27-
[Benchmark]
28-
public async Task SiskBFlat() => await _httpClient!.GetStringAsync("http://localhost:5111/");
2947
}

‎sisk-benchmark/sisk-benchmark.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net6.0</TargetFramework>
5+
<TargetFramework>net8.0</TargetFramework>
66
<RootNamespace>sisk_benchmark</RootNamespace>
77
<ImplicitUsings>enable</ImplicitUsings>
88
<Nullable>enable</Nullable>

‎sisk/Program.cs

+6-15
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,14 @@
11
using Sisk.Core.Http;
22
using Sisk.Core.Routing;
33

4-
namespace sisk;
5-
6-
internal class Program
4+
class Program
75
{
86
static void Main(string[] args)
97
{
10-
var response = new Sisk.Core.Http.HttpResponse(200).WithContent("Hello World!");
11-
var http = HttpServer.Emit(insecureHttpPort: 5222, out var config, out _, out var mainRouter);
12-
config.AccessLogsStream = null;
13-
config.Flags.NormalizeHeadersEncodings = false;
14-
config.Flags.SendSiskHeader = false;
15-
config.IncludeRequestIdHeader = false;
8+
var app = HttpServer.CreateBuilder(host => host.UseListeningPort(5222));
9+
10+
app.Router.SetRoute(RouteMethod.Get, "/", req => new HttpResponse().WithContent("Hello, world!"));
1611

17-
mainRouter += new Sisk.Core.Routing.Route(RouteMethod.Get, "/",
18-
request => response
19-
);
20-
http.Start();
21-
Thread.Sleep(-1);
12+
app.Start();
2213
}
23-
}
14+
}

‎sisk/Properties/launchSettings.json

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"profiles": {
3+
"sisk": {
4+
"commandName": "Project",
5+
"launchBrowser": true,
6+
"environmentVariables": {
7+
"ASPNETCORE_ENVIRONMENT": "Development"
8+
},
9+
"applicationUrl": "https://localhost:55298;http://localhost:55299"
10+
}
11+
}
12+
}

‎sisk/sisk.csproj

+11-10
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
1-
<Project Sdk="Microsoft.NET.Sdk">
1+
<Project Sdk="Microsoft.NET.Sdk">
22

3-
<PropertyGroup>
4-
<OutputType>Exe</OutputType>
5-
<TargetFramework>net6.0</TargetFramework>
6-
<ImplicitUsings>enable</ImplicitUsings>
7-
<Nullable>enable</Nullable>
8-
</PropertyGroup>
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>net8.0</TargetFramework>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<Nullable>enable</Nullable>
8+
<ServerGarbageCollection>true</ServerGarbageCollection>
9+
</PropertyGroup>
910

10-
<ItemGroup>
11-
<PackageReference Include="Sisk.HttpServer" Version="0.15.0.46-beta" />
12-
</ItemGroup>
11+
<ItemGroup>
12+
<ProjectReference Include="..\..\core\src\Sisk.Core.csproj" />
13+
</ItemGroup>
1314

1415
</Project>

0 commit comments

Comments
 (0)