Skip to content

Commit 926e323

Browse files
benbierensbenbierens
committed
Better file manager
1 parent 8d32758 commit 926e323

File tree

6 files changed

+201
-44
lines changed

6 files changed

+201
-44
lines changed

‎Framework/FileUtils/FileManager.cs

Lines changed: 35 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -7,27 +7,35 @@ public interface IFileManager
77
{
88
TrackedFile CreateEmptyFile(string label = "");
99
TrackedFile GenerateFile(ByteSize size, string label = "");
10+
TrackedFile GenerateFile(Action<IGenerateOption> options, string label = "");
1011
void DeleteAllFiles();
1112
void ScopedFiles(Action action);
1213
T ScopedFiles<T>(Func<T> action);
1314
}
1415

16+
public interface IGenerateOption
17+
{
18+
IGenerateOption Random(ByteSize size);
19+
IGenerateOption StringRepeat(string str, ByteSize size);
20+
IGenerateOption StringRepeat(string str, int times);
21+
IGenerateOption ByteRepeat(byte[] bytes, ByteSize size);
22+
IGenerateOption ByteRepeat(byte[] bytes, int times);
23+
}
24+
1525
public class FileManager : IFileManager
1626
{
17-
public const int ChunkSize = 1024 * 1024 * 100;
18-
private static NumberSource folderNumberSource = new NumberSource(0);
19-
private readonly Random random = new Random();
27+
private static readonly NumberSource folderNumberSource = new NumberSource(0);
2028
private readonly ILog log;
21-
private readonly string rootFolder;
2229
private readonly string folder;
2330
private readonly List<List<TrackedFile>> fileSetStack = new List<List<TrackedFile>>();
2431

32+
public const int ChunkSize = 1024 * 1024 * 100;
33+
2534
public FileManager(ILog log, string rootFolder)
2635
{
2736
folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5"));
2837

2938
this.log = log;
30-
this.rootFolder = rootFolder;
3139
}
3240

3341
public TrackedFile CreateEmptyFile(string label = "")
@@ -41,10 +49,15 @@ public TrackedFile CreateEmptyFile(string label = "")
4149
return result;
4250
}
4351

44-
public TrackedFile GenerateFile(ByteSize size, string label)
52+
public TrackedFile GenerateFile(ByteSize size, string label = "")
53+
{
54+
return GenerateFile(o => o.Random(size), label);
55+
}
56+
57+
public TrackedFile GenerateFile(Action<IGenerateOption> options, string label = "")
4558
{
4659
var sw = Stopwatch.Begin(log);
47-
var result = GenerateRandomFile(size, label);
60+
var result = RunGenerators(options, label);
4861
sw.End($"Generated file {result.Describe()}.");
4962
return result;
5063
}
@@ -89,61 +102,42 @@ private void PopFileSet()
89102
if (!Directory.GetFiles(folder).Any()) DeleteDirectory();
90103
}
91104

92-
private TrackedFile GenerateRandomFile(ByteSize size, string label)
105+
private TrackedFile RunGenerators(Action<IGenerateOption> options, string label)
93106
{
94107
var result = CreateEmptyFile(label);
95-
CheckSpaceAvailable(result, size);
108+
var generators = GetGenerators(options);
109+
CheckSpaceAvailable(result, generators.GetRequiredSpace());
96110

97-
GenerateFileBytes(result, size);
111+
using var stream = new FileStream(result.Filename, FileMode.Append);
112+
generators.Run(stream);
98113
return result;
99114
}
100115

101-
private void CheckSpaceAvailable(TrackedFile testFile, ByteSize size)
116+
private GeneratorCollection GetGenerators(Action<IGenerateOption> options)
117+
{
118+
var result = new GeneratorCollection();
119+
options(result);
120+
return result;
121+
}
122+
123+
private void CheckSpaceAvailable(TrackedFile testFile, long requiredSize)
102124
{
103125
var file = new FileInfo(testFile.Filename);
104126
var drive = new DriveInfo(file.Directory!.Root.FullName);
105127

106128
var spaceAvailable = drive.TotalFreeSpace;
107129

108-
if (spaceAvailable < size.SizeInBytes)
130+
if (spaceAvailable < requiredSize)
109131
{
110132
var msg = $"Not enough disk space. " +
111-
$"{Formatter.FormatByteSize(size.SizeInBytes)} required. " +
133+
$"{Formatter.FormatByteSize(requiredSize)} required. " +
112134
$"{Formatter.FormatByteSize(spaceAvailable)} available.";
113135

114136
log.Log(msg);
115137
throw new Exception(msg);
116138
}
117139
}
118140

119-
private void GenerateFileBytes(TrackedFile result, ByteSize size)
120-
{
121-
long bytesLeft = size.SizeInBytes;
122-
int chunkSize = ChunkSize;
123-
while (bytesLeft > 0)
124-
{
125-
try
126-
{
127-
var length = Math.Min(bytesLeft, chunkSize);
128-
AppendRandomBytesToFile(result, length);
129-
bytesLeft -= length;
130-
}
131-
catch
132-
{
133-
chunkSize = chunkSize / 2;
134-
if (chunkSize < 1024) throw;
135-
}
136-
}
137-
}
138-
139-
private void AppendRandomBytesToFile(TrackedFile result, long length)
140-
{
141-
var bytes = new byte[length];
142-
random.NextBytes(bytes);
143-
using var stream = new FileStream(result.Filename, FileMode.Append);
144-
stream.Write(bytes, 0, bytes.Length);
145-
}
146-
147141
private void EnsureDirectory()
148142
{
149143
Directory.CreateDirectory(folder);

‎Framework/FileUtils/Generators.cs

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
using System.Text;
2+
using Utils;
3+
4+
namespace FileUtils
5+
{
6+
public class GeneratorCollection : IGenerateOption
7+
{
8+
private readonly List<IGenerator> generators = new List<IGenerator>();
9+
10+
public IGenerateOption ByteRepeat(byte[] bytes, ByteSize size)
11+
{
12+
var times = size.SizeInBytes / bytes.Length;
13+
generators.Add(new ByteRepeater(bytes, times));
14+
return this;
15+
}
16+
17+
public IGenerateOption ByteRepeat(byte[] bytes, int times)
18+
{
19+
generators.Add(new ByteRepeater(bytes, times));
20+
return this;
21+
}
22+
23+
public IGenerateOption Random(ByteSize size)
24+
{
25+
generators.Add(new RandomGenerator(size));
26+
return this;
27+
}
28+
29+
public IGenerateOption StringRepeat(string str, ByteSize size)
30+
{
31+
var times = size.SizeInBytes / str.Length;
32+
generators.Add(new StringRepeater(str, times));
33+
return this;
34+
}
35+
36+
public IGenerateOption StringRepeat(string str, int times)
37+
{
38+
generators.Add(new StringRepeater(str, times));
39+
return this;
40+
}
41+
42+
public void Run(FileStream file)
43+
{
44+
foreach (var generator in generators)
45+
{
46+
generator.Generate(file);
47+
}
48+
}
49+
50+
public long GetRequiredSpace()
51+
{
52+
return generators.Sum(g => g.GetRequiredSpace());
53+
}
54+
}
55+
56+
public interface IGenerator
57+
{
58+
void Generate(FileStream file);
59+
long GetRequiredSpace();
60+
}
61+
62+
public class ByteRepeater : IGenerator
63+
{
64+
private readonly byte[] bytes;
65+
private readonly long times;
66+
67+
public ByteRepeater(byte[] bytes, long times)
68+
{
69+
this.bytes = bytes;
70+
this.times = times;
71+
}
72+
73+
public void Generate(FileStream file)
74+
{
75+
for (var i = 0; i < times; i++)
76+
{
77+
file.Write(bytes, 0, bytes.Length);
78+
}
79+
}
80+
81+
public long GetRequiredSpace()
82+
{
83+
return bytes.Length * times;
84+
}
85+
}
86+
87+
public class StringRepeater : IGenerator
88+
{
89+
private readonly string str;
90+
private readonly long times;
91+
92+
public StringRepeater(string str, long times)
93+
{
94+
this.str = str;
95+
this.times = times;
96+
}
97+
98+
public void Generate(FileStream file)
99+
{
100+
using var writer = new StreamWriter(file);
101+
for (var i = 0; i < times; i++)
102+
{
103+
writer.Write(str);
104+
}
105+
}
106+
107+
public long GetRequiredSpace()
108+
{
109+
return Encoding.ASCII.GetBytes(str).Length * times;
110+
}
111+
}
112+
113+
public class RandomGenerator : IGenerator
114+
{
115+
private readonly Random random = new Random();
116+
private readonly ByteSize size;
117+
118+
public RandomGenerator(ByteSize size)
119+
{
120+
this.size = size;
121+
}
122+
123+
public void Generate(FileStream file)
124+
{
125+
var bytesLeft = size.SizeInBytes;
126+
while (bytesLeft > 0)
127+
{
128+
var size = Math.Min(bytesLeft, FileManager.ChunkSize);
129+
var bytes = new byte[size];
130+
random.NextBytes(bytes);
131+
file.Write(bytes, 0, bytes.Length);
132+
bytesLeft -= size;
133+
}
134+
}
135+
136+
public long GetRequiredSpace()
137+
{
138+
return size.SizeInBytes;
139+
}
140+
}
141+
}

‎Tests/CodexContinuousTests/Tests/TwoClientTest.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
using CodexPlugin;
22
using FileUtils;
33
using Logging;
4-
using Newtonsoft.Json;
54
using NUnit.Framework;
65
using Utils;
76

‎Tests/CodexTests/BasicTests/MarketplaceTests.cs

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using CodexContractsPlugin;
22
using CodexContractsPlugin.Marketplace;
33
using CodexPlugin;
4+
using FileUtils;
45
using GethPlugin;
56
using NUnit.Framework;
67
using Utils;
@@ -46,7 +47,7 @@ public void MarketplaceExample()
4647
host.Marketplace.MakeStorageAvailable(availability);
4748
}
4849

49-
var testFile = GenerateTestFile(fileSize);
50+
var testFile = CreateFile(fileSize);
5051

5152
var client = StartCodex(s => s
5253
.WithName("Client")
@@ -90,7 +91,7 @@ public void CanDownloadContentFromContractCid()
9091
var fileSize = 10.MB();
9192
var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
9293
var contracts = Ci.StartCodexContracts(geth);
93-
var testFile = GenerateTestFile(fileSize);
94+
var testFile = CreateFile(fileSize);
9495

9596
var client = StartCodex(s => s
9697
.WithName("Client")
@@ -124,6 +125,18 @@ public void CanDownloadContentFromContractCid()
124125
testFile.AssertIsEqual(downloader.DownloadContent(contractCid));
125126
}
126127

128+
private TrackedFile CreateFile(ByteSize fileSize)
129+
{
130+
var segmentSize = new ByteSize(fileSize.SizeInBytes / 4);
131+
132+
return GenerateTestFile(o => o
133+
.Random(segmentSize)
134+
.ByteRepeat(new byte[] { 0xaa }, segmentSize)
135+
.Random(segmentSize)
136+
.ByteRepeat(new byte[] { 0xee }, segmentSize)
137+
);
138+
}
139+
127140
private void WaitForAllSlotFilledEvents(ICodexContracts contracts, StoragePurchaseRequest purchase, IGethNode geth)
128141
{
129142
Time.Retry(() =>

‎Tests/DistTestCore/DistTest.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,11 @@ public TrackedFile GenerateTestFile(ByteSize size, string label = "")
117117
return Get().GenerateTestFile(size, label);
118118
}
119119

120+
public TrackedFile GenerateTestFile(Action<IGenerateOption> options, string label = "")
121+
{
122+
return Get().GenerateTestFile(options, label);
123+
}
124+
120125
/// <summary>
121126
/// Any test files generated in 'action' will be deleted after it returns.
122127
/// This helps prevent large tests from filling up discs.

‎Tests/DistTestCore/TestLifecycle.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@ public TrackedFile GenerateTestFile(ByteSize size, string label = "")
5252
return entryPoint.Tools.GetFileManager().GenerateFile(size, label);
5353
}
5454

55+
public TrackedFile GenerateTestFile(Action<IGenerateOption> options, string label = "")
56+
{
57+
return entryPoint.Tools.GetFileManager().GenerateFile(options, label);
58+
}
59+
5560
public IFileManager GetFileManager()
5661
{
5762
return entryPoint.Tools.GetFileManager();

0 commit comments

Comments
 (0)