635 lines
34 KiB
ObjectPascal
635 lines
34 KiB
ObjectPascal
{ Read-only raw storage benchmark. No disk I/O is performed by UnitTest. }
|
|
unit driver.storage.bench;
|
|
interface
|
|
procedure init;
|
|
procedure UnitTest;
|
|
implementation
|
|
uses boot.mgr, io.stdio, io.syslog, core.strings, core.util, arch.x86.util,
|
|
arch.x86.bda, arch.x86.isr.tmr0, memory.heap, proc.mgr, proc.types,
|
|
driver.storage.types, driver.storage.mgr, core.ds.cfifo, driver.storage.bench.clock;
|
|
const
|
|
MAX_DEPTH = 32;
|
|
DEFAULT_MIB = 128;
|
|
AUTO_MIB = 32;
|
|
MIN_SAMPLE_US = 10000; { avoid presenting millisecond cache hits as drive ratings }
|
|
type
|
|
PCaseResult = ^TCaseResult;
|
|
TCaseResult = record
|
|
Name : pchar;
|
|
KiB, Depth : uint32;
|
|
RandomRead, Measured : boolean;
|
|
Error : TError;
|
|
Rate10, IOPS, AvgUS, P95US, Bytes : uint64;
|
|
Elapsed : uint64;
|
|
end;
|
|
PBench = ^TBench;
|
|
PSlot = ^TSlot;
|
|
TSlot = record
|
|
Owner : PBench;
|
|
Buffer : puint32;
|
|
State, LBA, Sectors : uint32; { State: 0 free, 1 DMA, 2 completed }
|
|
Started : uint64;
|
|
end;
|
|
TBench = record
|
|
Device : PStorage_Device;
|
|
Process : PProcessContext;
|
|
Buffer : pointer;
|
|
Depth, RequestBytes, ChunkSectors, StartLBA, Span, Remaining, Seed : uint32;
|
|
RandomRead : boolean;
|
|
Active, Peak, WakeSequence, Accepted, Completed, Successful : uint32;
|
|
Error : TError;
|
|
ErrorLBA : uint32;
|
|
StartTick, Elapsed, LatencyMax : uint64;
|
|
Bytes, LatencyTotal : uint64;
|
|
Histogram : array[0..31] of uint32;
|
|
BeforeStats, AfterStats : TStorageStats;
|
|
Slots : array[0..MAX_DEPTH-1] of TSlot;
|
|
end;
|
|
|
|
function nowTick : uint64;
|
|
begin nowTick := driver.storage.bench.clock.nowUS; end;
|
|
|
|
{ Reject overflow before multiplication; CLI parsing must never wrap. }
|
|
function parseNumber(text : pchar; limit : uint32; var value : uint32) : boolean;
|
|
var i, digit : uint32;
|
|
begin
|
|
parseNumber := false; value := 0;
|
|
if (text=nil) or (text[0]=#0) then exit;
|
|
i := 0;
|
|
while text[i]<>#0 do begin
|
|
if (text[i]<'0') or (text[i]>'9') then exit;
|
|
digit := ord(text[i])-ord('0');
|
|
if (value>limit div 10) or ((value=limit div 10) and (digit>limit mod 10)) then exit;
|
|
value := value*10+digit; Inc(i);
|
|
end;
|
|
parseNumber := true;
|
|
end;
|
|
|
|
function ticksToUS(ticks : uint64) : uint64;
|
|
begin ticksToUS := ticks; end; { samples now use independent hardware microseconds }
|
|
function perSecond(value, ticks : uint64) : uint64;
|
|
begin
|
|
perSecond := 0;
|
|
if ticks<>0 then perSecond := (value*1000000) div ticks;
|
|
end;
|
|
function latencyBucket(ticks : uint64) : uint32;
|
|
var bucket : uint32;
|
|
begin
|
|
bucket := 0;
|
|
while (ticks<>0) and (bucket<31) do begin Inc(bucket); ticks := ticks shr 1; end;
|
|
latencyBucket := bucket;
|
|
end;
|
|
function p95Ticks(run : PBench) : uint64;
|
|
var target, count, i : uint32;
|
|
begin
|
|
p95Ticks := 0;
|
|
if run^.Completed=0 then exit;
|
|
target := run^.Completed - run^.Completed div 20; count := 0;
|
|
for i := 0 to 31 do begin
|
|
Inc(count,run^.Histogram[i]);
|
|
if count>=target then begin
|
|
if i=31 then p95Ticks := run^.LatencyMax
|
|
else p95Ticks := (uint32(1) shl i)-1;
|
|
exit;
|
|
end;
|
|
end;
|
|
end;
|
|
|
|
procedure writeNumber(output : POutBuf; value : uint64);
|
|
var digits : array[0..20] of char; n : uint32;
|
|
begin
|
|
n := 0;
|
|
repeat digits[n] := char(ord('0')+value mod 10); value := value div 10; Inc(n); until value=0;
|
|
while n>0 do begin Dec(n); bufWriteChar(output,digits[n]); end;
|
|
end;
|
|
procedure writeValue(output : POutBuf; name : pchar; value : uint64);
|
|
begin bufWriteStr(output,name); writeNumber(output,value); end;
|
|
|
|
{ All geometry arithmetic is bounded before buffer allocation or I/O. }
|
|
function configure(run : PBench; device : PStorage_Device; mib, kib, depth, start : uint32;
|
|
explicitStart, randomRead : boolean) : boolean;
|
|
var sectors : uint64;
|
|
begin
|
|
configure := false;
|
|
if device=nil then exit;
|
|
if device^.removed or (device^.dispatchRead=nil) or (device^.maxSectorCount=0) then exit;
|
|
if (device^.sectorSize=0) or (device^.sectorSize>1048576) then exit;
|
|
if (mib=0) or (mib>16384) or (kib=0) or (kib>1024) or (depth=0) or (depth>MAX_DEPTH) then exit;
|
|
if ((kib*1024) mod device^.sectorSize)<>0 then exit;
|
|
sectors := (uint64(mib) shl 20) div device^.sectorSize;
|
|
if not explicitStart then begin
|
|
start := 0;
|
|
if device^.maxSectorCount>2048 then
|
|
if sectors<=device^.maxSectorCount-2048 then start := 2048;
|
|
end;
|
|
if start>=device^.maxSectorCount then exit;
|
|
if sectors>device^.maxSectorCount-start then sectors := device^.maxSectorCount-start;
|
|
if sectors=0 then exit;
|
|
run^.Device := device; run^.Depth := depth; run^.RequestBytes := kib*1024;
|
|
run^.ChunkSectors := run^.RequestBytes div device^.sectorSize;
|
|
run^.StartLBA := start; run^.Span := uint32(sectors); run^.Remaining := run^.Span;
|
|
run^.Seed := $6D2B79F5; run^.RandomRead := randomRead;
|
|
configure := true;
|
|
end;
|
|
|
|
procedure planRead(run : PBench; slot : PSlot);
|
|
var span : uint32;
|
|
begin
|
|
slot^.Sectors := run^.ChunkSectors;
|
|
if slot^.Sectors>run^.Remaining then slot^.Sectors := run^.Remaining;
|
|
if run^.RandomRead then begin
|
|
run^.Seed := run^.Seed xor (run^.Seed shl 13);
|
|
run^.Seed := run^.Seed xor (run^.Seed shr 17);
|
|
run^.Seed := run^.Seed xor (run^.Seed shl 5);
|
|
span := run^.Span-slot^.Sectors+1;
|
|
slot^.LBA := run^.StartLBA + run^.Seed mod span;
|
|
end else slot^.LBA := run^.StartLBA + (run^.Span-run^.Remaining);
|
|
Dec(run^.Remaining,slot^.Sectors);
|
|
end;
|
|
|
|
procedure readDone(error : TError; data : pointer);
|
|
var slot : PSlot; run : PBench; ticks : uint64;
|
|
begin
|
|
slot := PSlot(data); run := slot^.Owner;
|
|
ticks := nowTick-slot^.Started;
|
|
Inc(run^.Completed); Inc(run^.LatencyTotal,ticks);
|
|
if ticks>run^.LatencyMax then run^.LatencyMax := ticks;
|
|
Inc(run^.Histogram[latencyBucket(ticks)]);
|
|
if error=eNone then begin
|
|
Inc(run^.Successful); Inc(run^.Bytes,uint64(slot^.Sectors)*run^.Device^.sectorSize);
|
|
end else if run^.Error=eNone then begin run^.Error := error; run^.ErrorLBA := slot^.LBA; end;
|
|
Dec(run^.Active); slot^.State := 2;
|
|
if run^.Process<>nil then run^.Process^.State := psReady;
|
|
Inc(run^.WakeSequence);
|
|
end;
|
|
|
|
{ Task context only. Reserve before submit to support immediate callbacks.
|
|
Stop submitting on the first error, but drain accepted DMA before freeing. }
|
|
procedure execute(run : PBench);
|
|
var i, flags, wake, attemptedActive : uint32; slot : PSlot; err : TError; finished, ready : boolean;
|
|
begin
|
|
run^.Process := CurrentProcess;
|
|
for i := 0 to run^.Depth-1 do begin
|
|
run^.Slots[i].Owner := run;
|
|
run^.Slots[i].Buffer := puint32(uint32(run^.Buffer)+i*run^.RequestBytes);
|
|
end;
|
|
storage_get_stats(run^.Device,run^.BeforeStats); run^.StartTick := nowTick;
|
|
repeat
|
|
for i := 0 to run^.Depth-1 do begin
|
|
asm pushfd; cli; pop flags end;
|
|
slot := @run^.Slots[i];
|
|
if slot^.State=2 then slot^.State := 0;
|
|
if (slot^.State=0) and (run^.Remaining>0) and (run^.Error=eNone) then begin
|
|
planRead(run,slot); slot^.State := 1; slot^.Started := nowTick;
|
|
Inc(run^.Active); Inc(run^.Accepted); attemptedActive := run^.Active;
|
|
err := storage_read_async(run^.Device,slot^.LBA,slot^.Sectors,slot^.Buffer,@readDone,slot);
|
|
if err<>eNone then begin
|
|
Dec(run^.Active); Dec(run^.Accepted); slot^.State := 0;
|
|
if run^.Error=eNone then begin run^.Error := err; run^.ErrorLBA := slot^.LBA; end;
|
|
end;
|
|
if (err=eNone) and (attemptedActive>run^.Peak) then run^.Peak := attemptedActive;
|
|
end;
|
|
if (flags and $200)<>0 then asm sti end;
|
|
end;
|
|
asm pushfd; cli; pop flags end;
|
|
finished := (run^.Active=0) and ((run^.Remaining=0) or (run^.Error<>eNone));
|
|
ready := false;
|
|
for i := 0 to run^.Depth-1 do if run^.Slots[i].State=2 then ready := true;
|
|
wake := run^.WakeSequence;
|
|
if not finished and not ready and (run^.Process<>nil) then run^.Process^.State := psAwaiting;
|
|
if (flags and $200)<>0 then asm sti end;
|
|
if not finished and not ready then
|
|
while puint32(@run^.WakeSequence)^=wake do asm hlt end;
|
|
until finished;
|
|
run^.Elapsed := nowTick-run^.StartTick;
|
|
storage_get_stats(run^.Device,run^.AfterStats);
|
|
end;
|
|
|
|
procedure report(run : PBench; output, errors : POutBuf);
|
|
var rate, ticks, commands : uint64;
|
|
begin
|
|
if run^.Error<>eNone then begin
|
|
writeValue(errors,'Read failed at LBA ',run^.ErrorLBA);
|
|
writeValue(errors,', error=',ord(run^.Error));
|
|
bufWriteStrLn(errors,'. All accepted reads drained; results below are partial.');
|
|
end;
|
|
writeValue(output,' Read ',run^.Bytes div 1024); bufWriteStr(output,' KiB');
|
|
writeValue(output,' in ',ticksToUS(run^.Elapsed)); bufWriteStr(output,' us; ');
|
|
if run^.Elapsed<MIN_SAMPLE_US then bufWriteStrLn(output,'sample shorter than 10 ms; use a larger disk/range.')
|
|
else begin
|
|
rate := perSecond(run^.Bytes*10,run^.Elapsed) div 1048576;
|
|
writeNumber(output,rate div 10); bufWriteChar(output,'.'); writeNumber(output,rate mod 10);
|
|
bufWriteStr(output,' MiB/s; '); writeNumber(output,perSecond(run^.Successful,run^.Elapsed)); bufWriteStrLn(output,' IOPS');
|
|
end;
|
|
writeValue(output,' Requests accepted/completed/successful: ',run^.Accepted);
|
|
writeValue(output,' / ',run^.Completed); writeValue(output,' / ',run^.Successful);
|
|
writeValue(output,'; peak outstanding: ',run^.Peak); bufWriteNewLine(output);
|
|
if run^.Completed>0 then begin
|
|
writeValue(output,' Latency avg: ',ticksToUS(run^.LatencyTotal) div run^.Completed);
|
|
ticks := p95Ticks(run);
|
|
if ticks=0 then writeValue(output,' us; p95 < ',ticksToUS(1)+1)
|
|
else writeValue(output,' us; p95 < ',ticksToUS(ticks+1)+1);
|
|
writeValue(output,' us; max < ',ticksToUS(uint64(run^.LatencyMax)+1)+1);
|
|
bufWriteStrLn(output,' us (hardware clock; p95 uses log buckets)');
|
|
end;
|
|
commands := uint32(run^.AfterStats.Commands-run^.BeforeStats.Commands);
|
|
writeValue(output,' Device deltas: commands=',commands);
|
|
writeValue(output,', merged=',uint32(run^.AfterStats.Merged-run^.BeforeStats.Merged));
|
|
writeValue(output,', failed=',uint32(run^.AfterStats.Failed-run^.BeforeStats.Failed));
|
|
writeValue(output,', queue-full=',uint32(run^.AfterStats.QueueFull-run^.BeforeStats.QueueFull));
|
|
bufWriteNewLine(output);
|
|
|
|
end;
|
|
|
|
function runCase(device : PStorage_Device; mib, kib, depth, start : uint32;
|
|
explicitStart, randomRead : boolean; output, errors : POutBuf;
|
|
summary : PCaseResult) : boolean;
|
|
var run : PBench;
|
|
begin
|
|
runCase := false;
|
|
run := PBench(kalloc(sizeof(TBench)));
|
|
if summary<>nil then summary^.Error := eOutOfMemory;
|
|
if run=nil then begin bufWriteStrLn(errors,'Out of memory allocating benchmark state.'); exit; end;
|
|
memset(uint32(run),0,sizeof(TBench));
|
|
if not configure(run,device,mib,kib,depth,start,explicitStart,randomRead) then begin
|
|
if summary<>nil then summary^.Error := eInvalidArgument;
|
|
bufWriteStrLn(errors,'Invalid/unreadable device, range, or request size (must be a whole number of device sectors).');
|
|
kfree(void(run)); exit;
|
|
end;
|
|
run^.Buffer := kalloc(run^.Depth*run^.RequestBytes);
|
|
if run^.Buffer=nil then begin bufWriteStrLn(errors,'Out of memory allocating benchmark buffers.'); kfree(void(run)); exit; end;
|
|
memset(uint32(run^.Buffer),0,run^.Depth*run^.RequestBytes);
|
|
if summary=nil then begin
|
|
if randomRead then bufWriteStr(output,'Random') else bufWriteStr(output,'Sequential');
|
|
writeValue(output,' read: ',kib); writeValue(output,' KiB/request, depth=',depth);
|
|
writeValue(output,', LBA=',run^.StartLBA);
|
|
writeValue(output,', span=',uint64(run^.Span)*device^.sectorSize div 1024); bufWriteStrLn(output,' KiB');
|
|
end;
|
|
execute(run); runCase := run^.Error=eNone;
|
|
if summary=nil then report(run,output,errors)
|
|
else begin
|
|
summary^.Error := run^.Error; summary^.Bytes := run^.Bytes;
|
|
summary^.Elapsed := run^.Elapsed; summary^.Measured := run^.Elapsed>=MIN_SAMPLE_US;
|
|
if summary^.Measured then begin
|
|
summary^.Rate10 := perSecond(run^.Bytes*10,run^.Elapsed) div 1048576;
|
|
summary^.IOPS := perSecond(run^.Successful,run^.Elapsed);
|
|
end;
|
|
if run^.Completed>0 then begin
|
|
summary^.AvgUS := ticksToUS(run^.LatencyTotal) div run^.Completed;
|
|
summary^.P95US := ticksToUS(uint64(p95Ticks(run))+1)+1;
|
|
end;
|
|
if run^.Error<>eNone then begin
|
|
writeValue(errors,'Read failed at LBA ',run^.ErrorLBA);
|
|
writeValue(errors,', error=',ord(run^.Error)); bufWriteStrLn(errors,'. Accepted reads drained.');
|
|
end;
|
|
end;
|
|
kfree(run^.Buffer); kfree(void(run));
|
|
end;
|
|
|
|
{ Fixed-width ASCII output stays readable in terminals and redirected logs. }
|
|
procedure textCell(output : POutBuf; text : pchar; width : uint32);
|
|
var i : uint32;
|
|
begin
|
|
i := 0;
|
|
while (i<width) and (text[i]<>#0) do begin bufWriteChar(output,text[i]); Inc(i); end;
|
|
while i<width do begin bufWriteChar(output,' '); Inc(i); end;
|
|
end;
|
|
procedure numberCell(output : POutBuf; value : uint64; width : uint32; decimal : boolean);
|
|
var copyValue : uint64; digits : uint32;
|
|
begin
|
|
copyValue := value; digits := 0;
|
|
repeat Inc(digits); copyValue := copyValue div 10; until copyValue=0;
|
|
if decimal then begin if digits<2 then digits:=2; Inc(digits); end;
|
|
while digits<width do begin bufWriteChar(output,' '); Inc(digits); end;
|
|
if decimal then begin writeNumber(output,value div 10); bufWriteChar(output,'.'); writeNumber(output,value mod 10); end
|
|
else writeNumber(output,value);
|
|
end;
|
|
procedure tableLine(output : POutBuf);
|
|
begin bufWriteStrLn(output,'-------------------------------------------------------------------------------'); end;
|
|
procedure tableHeader(output : POutBuf);
|
|
begin
|
|
tableLine(output);
|
|
textCell(output,'Workload',18); textCell(output,' KiB',6); textCell(output,' QD',4);
|
|
textCell(output,' MiB/s',10); textCell(output,' IOPS',10);
|
|
textCell(output,' Avg us',10); textCell(output,' P95< us',10); bufWriteStrLn(output,' Status');
|
|
tableLine(output);
|
|
end;
|
|
procedure tableRow(output : POutBuf; summary : PCaseResult);
|
|
begin
|
|
textCell(output,summary^.Name,18); numberCell(output,summary^.KiB,6,false);
|
|
numberCell(output,summary^.Depth,4,false);
|
|
if (summary^.Error=eNone) and summary^.Measured then begin
|
|
numberCell(output,summary^.Rate10,10,true); numberCell(output,summary^.IOPS,10,false);
|
|
numberCell(output,summary^.AvgUS,10,false); numberCell(output,summary^.P95US,10,false);
|
|
bufWriteStrLn(output,' OK');
|
|
end else begin
|
|
bufWriteStr(output,' - - - -');
|
|
if summary^.Error<>eNone then bufWriteStrLn(output,' FAILED') else bufWriteStrLn(output,' SHORT');
|
|
end;
|
|
end;
|
|
function suiteProfile(index : uint32; automatic : boolean; hardwareDepth : uint32;
|
|
summary : PCaseResult) : boolean;
|
|
var queued : uint32;
|
|
begin
|
|
suiteProfile := false;
|
|
memset(uint32(summary),0,sizeof(TCaseResult));
|
|
queued := hardwareDepth; if queued<4 then queued:=4; if queued>16 then queued:=16;
|
|
if automatic then begin
|
|
if index>=6 then exit;
|
|
case index of
|
|
0:begin summary^.Name:='Seq large'; summary^.KiB:=1024; summary^.Depth:=1; end;
|
|
1:begin summary^.Name:='Seq queued'; summary^.KiB:=1024; summary^.Depth:=queued; end;
|
|
2:begin summary^.Name:='Seq medium'; summary^.KiB:=64; summary^.Depth:=queued; end;
|
|
3:begin summary^.Name:='Random small'; summary^.KiB:=4; summary^.Depth:=1; end;
|
|
4:begin summary^.Name:='Random queued'; summary^.KiB:=4; summary^.Depth:=queued; end;
|
|
5:begin summary^.Name:='Random medium'; summary^.KiB:=64; summary^.Depth:=queued; end;
|
|
end;
|
|
summary^.RandomRead := index>=3;
|
|
end else begin
|
|
if index>=12 then exit;
|
|
summary^.Name := 'Sequential';
|
|
case index div 4 of 0:summary^.KiB:=4; 1:summary^.KiB:=64; 2:summary^.KiB:=1024; end;
|
|
case index mod 4 of 0:summary^.Depth:=1; 1:summary^.Depth:=4; 2:summary^.Depth:=16; 3:summary^.Depth:=32; end;
|
|
end;
|
|
suiteProfile := true;
|
|
end;
|
|
procedure queueGain(output : POutBuf; name : pchar; baseline, queued : PCaseResult);
|
|
begin
|
|
if (baseline^.Error<>eNone) or (queued^.Error<>eNone) or
|
|
not baseline^.Measured or not queued^.Measured or (baseline^.Rate10=0) then exit;
|
|
bufWriteStr(output,name); numberCell(output,(queued^.Rate10*10) div baseline^.Rate10,0,true);
|
|
bufWriteStrLn(output,'x QD1 throughput');
|
|
end;
|
|
procedure runSuite(device : PStorage_Device; disk, mib : uint32; automatic : boolean;
|
|
output, errors : POutBuf);
|
|
var summaries, row : PCaseResult; count, i, completed, bestSeq, bestRandom : uint32;
|
|
totalBytes, totalTicks : uint64; ok : boolean;
|
|
begin
|
|
if automatic then count:=6 else count:=12;
|
|
summaries := PCaseResult(kalloc(count*sizeof(TCaseResult)));
|
|
if summaries=nil then begin bufWriteStrLn(errors,'Out of memory allocating results.'); exit; end;
|
|
memset(uint32(summaries),0,count*sizeof(TCaseResult));
|
|
bufWriteNewLine(output); tableLine(output);
|
|
if automatic then bufWriteStrLn(output,' STORAGE BENCHMARK / AUTO') else bufWriteStrLn(output,' STORAGE BENCHMARK / COMPARE');
|
|
writeValue(output,' Disk ',disk); writeValue(output,' | ',mib); writeValue(output,' MiB/profile | ',count);
|
|
writeValue(output,' profiles | hardware QD ',device^.maxActive); bufWriteNewLine(output);
|
|
bufWriteStrLn(output,' Raw read throughput (includes host/drive cache effects).');
|
|
bufWriteStrLn(output,' Asuro filesystem cache bypassed. Host/drive caches are not flushed.');
|
|
bufWriteStrLn(output,' Profiles reuse the same range; later reads may be warm-cache hits.');
|
|
writeValue(output,' Device capacity: ',(uint64(device^.maxSectorCount)*device^.sectorSize) div 1048576); bufWriteStrLn(output,' MiB');
|
|
if (uint64(mib) shl 20)>uint64(device^.maxSectorCount)*device^.sectorSize then
|
|
bufWriteStrLn(output,' Requested size exceeds this disk; each profile is clamped to capacity.');
|
|
tableHeader(output);
|
|
completed:=0; bestSeq:=$FFFFFFFF; bestRandom:=$FFFFFFFF; totalBytes:=0; totalTicks:=0;
|
|
for i:=0 to count-1 do begin
|
|
row:=@summaries[i];
|
|
if not suiteProfile(i,automatic,device^.maxActive,row) then break;
|
|
{ Keep the small-request profiles usable on larger-sector devices. }
|
|
if (row^.KiB*1024<device^.sectorSize) and (device^.sectorSize<=1048576) then
|
|
row^.KiB:=(device^.sectorSize+1023) div 1024;
|
|
ok:=runCase(device,mib,row^.KiB,row^.Depth,0,false,row^.RandomRead,output,errors,row);
|
|
tableRow(output,row); Inc(totalBytes,row^.Bytes); Inc(totalTicks,row^.Elapsed);
|
|
if not ok then break;
|
|
Inc(completed);
|
|
if row^.Measured then begin
|
|
if not row^.RandomRead then begin
|
|
if bestSeq=$FFFFFFFF then bestSeq:=i
|
|
else if row^.Rate10>summaries[bestSeq].Rate10 then bestSeq:=i;
|
|
end else begin
|
|
if bestRandom=$FFFFFFFF then bestRandom:=i
|
|
else if row^.IOPS>summaries[bestRandom].IOPS then bestRandom:=i;
|
|
end;
|
|
end;
|
|
end;
|
|
tableLine(output);
|
|
if bestSeq<>$FFFFFFFF then begin
|
|
bufWriteStr(output,' Best sequential: '); numberCell(output,summaries[bestSeq].Rate10,0,true);
|
|
writeValue(output,' MiB/s (',summaries[bestSeq].KiB); writeValue(output,' KiB, QD ',summaries[bestSeq].Depth); bufWriteStrLn(output,')');
|
|
end;
|
|
if bestRandom<>$FFFFFFFF then begin
|
|
writeValue(output,' Peak random: ',summaries[bestRandom].IOPS); bufWriteStr(output,' IOPS (');
|
|
writeNumber(output,summaries[bestRandom].KiB); writeValue(output,' KiB, QD ',summaries[bestRandom].Depth); bufWriteStrLn(output,')');
|
|
end;
|
|
if automatic then begin
|
|
if completed>=2 then queueGain(output,' Sequential queue gain: ',@summaries[0],@summaries[1]);
|
|
if completed>=5 then queueGain(output,' Small random queue gain: ',@summaries[3],@summaries[4]);
|
|
end;
|
|
writeValue(output,' Finished ',completed); writeValue(output,'/',count);
|
|
writeValue(output,' profiles | read ',totalBytes div 1024);
|
|
writeValue(output,' KiB | measured ',ticksToUS(totalTicks) div 1000); bufWriteStrLn(output,' ms');
|
|
if completed<count then bufWriteStrLn(output,' Suite stopped after an error; remaining profiles were not run.');
|
|
bufWriteStrLn(output,' P95 is a microsecond-bucket bound. SHORT means less than 10 ms.');
|
|
bufWriteStrLn(output,' Virtual/cached/sparse reads can exceed physical drive speed.');
|
|
tableLine(output); kfree(void(summaries));
|
|
end;
|
|
|
|
type TBenchMode = (bmAuto, bmCompare, bmManual, bmList, bmHelp);
|
|
function commandMode(first : pchar; count : uint32; var base : uint32) : TBenchMode;
|
|
begin
|
|
base:=0; commandMode:=bmAuto;
|
|
if count=0 then exit;
|
|
if stringEqualsCI(first,'help') then commandMode:=bmHelp
|
|
else if stringEqualsCI(first,'list') then commandMode:=bmList
|
|
else if stringEqualsCI(first,'auto') then begin commandMode:=bmAuto; base:=1; end
|
|
else if stringEqualsCI(first,'compare') then begin commandMode:=bmCompare; base:=1; end
|
|
else if stringEqualsCI(first,'run') then begin commandMode:=bmManual; base:=1; end
|
|
else if count>2 then commandMode:=bmManual; { retain detailed numeric syntax }
|
|
end;
|
|
procedure usage(output : POutBuf);
|
|
begin
|
|
bufWriteStrLn(output,'STORBENCH [disk] [MiB] Default six-profile suite');
|
|
bufWriteStrLn(output,'STORBENCH auto [disk] [MiB] Same suite (default 32 MiB/profile)');
|
|
bufWriteStrLn(output,'STORBENCH list List disks and capacities');
|
|
bufWriteStrLn(output,'STORBENCH compare [disk] [MiB] Twelve sequential cases');
|
|
bufWriteStrLn(output,'STORBENCH run [disk] [MiB] [KiB/request] [depth] [seq|random] [start_lba]');
|
|
bufWriteStrLn(output,'Examples: STORBENCH 2 512 STORBENCH run 2 512 64 4');
|
|
bufWriteStrLn(output,'Manual defaults: disk 0, 128 MiB, 1024 KiB/request, depth up to 4, sequential.');
|
|
bufWriteStrLn(output,'Limits: 1..16384 MiB, 1..1024 KiB/request, depth 1..32; sizes clamp to disk.');
|
|
bufWriteStrLn(output,'Read-only. Cached/sparse VM reads do not measure physical media speed.');
|
|
end;
|
|
procedure listDisks(output : POutBuf);
|
|
var count, i : uint32; device : PStorage_Device;
|
|
begin
|
|
count:=get_device_count;
|
|
if count=0 then begin bufWriteStrLn(output,'No storage devices.'); exit; end;
|
|
bufWriteStrLn(output,'Disk Controller MiB Sector bytes HW QD');
|
|
tableLine(output);
|
|
for i:=0 to count-1 do begin
|
|
device:=get_device(i);
|
|
if device<>nil then begin
|
|
numberCell(output,i,4,false); bufWriteStr(output,' ');
|
|
textCell(output,controller_type_2_string(device^.controller),16);
|
|
numberCell(output,(uint64(device^.maxSectorCount)*device^.sectorSize) div 1048576,9,false);
|
|
numberCell(output,device^.sectorSize,17,false); numberCell(output,device^.maxActive,7,false);
|
|
if device^.removed then bufWriteStr(output,' removed');
|
|
bufWriteNewLine(output);
|
|
end;
|
|
end;
|
|
bufWriteStrLn(output,'Select a disk and MiB per profile: STORBENCH 2 512');
|
|
end;
|
|
procedure command(params : PParamList; stdin_buf, stdout_buf, stderr_buf : POutBuf);
|
|
var count, disk, mib, kib, depth, start, base : uint32;
|
|
mode : TBenchMode; randomRead, explicitStart : boolean; device : PStorage_Device; first : pchar;
|
|
begin
|
|
count:=paramCount(params); first:=nil;
|
|
if count>0 then first:=getParam(0,params);
|
|
mode:=commandMode(first,count,base);
|
|
if mode=bmHelp then begin usage(stdout_buf); exit; end;
|
|
if mode=bmList then begin
|
|
if count<>1 then usage(stderr_buf) else listDisks(stdout_buf);
|
|
exit;
|
|
end;
|
|
if ((mode<>bmManual) and (count>base+2)) or ((mode=bmManual) and (count>base+6)) then begin usage(stderr_buf); exit; end;
|
|
disk:=0; mib:=DEFAULT_MIB; kib:=1024; start:=0;
|
|
if mode=bmAuto then mib:=AUTO_MIB;
|
|
randomRead:=false; explicitStart:=false;
|
|
if count>base then if not parseNumber(getParam(base,params),$FFFFFFFF,disk) then begin usage(stderr_buf); exit; end;
|
|
if count>base+1 then if not parseNumber(getParam(base+1,params),16384,mib) or (mib=0) then begin usage(stderr_buf); exit; end;
|
|
device:=get_device(disk);
|
|
if device=nil then begin bufWriteStrLn(stderr_buf,'Storage device not found. Use STORBENCH list.'); exit; end;
|
|
depth:=device^.maxActive; if depth>4 then depth:=4; if depth=0 then depth:=1;
|
|
if mode=bmManual then begin
|
|
if count>base+2 then if not parseNumber(getParam(base+2,params),1024,kib) or (kib=0) then begin usage(stderr_buf); exit; end;
|
|
if count>base+3 then if not parseNumber(getParam(base+3,params),MAX_DEPTH,depth) or (depth=0) then begin usage(stderr_buf); exit; end;
|
|
if count>base+4 then begin
|
|
randomRead:=stringEqualsCI(getParam(base+4,params),'random');
|
|
if not randomRead and not stringEqualsCI(getParam(base+4,params),'seq') then begin usage(stderr_buf); exit; end;
|
|
end;
|
|
if count>base+5 then begin
|
|
explicitStart:=true;
|
|
if not parseNumber(getParam(base+5,params),$FFFFFFFF,start) then begin usage(stderr_buf); exit; end;
|
|
end;
|
|
end;
|
|
if not driver.storage.bench.clock.prepare then begin
|
|
bufWriteStr(stderr_buf,'Benchmark clock unavailable: ');
|
|
bufWriteStrLn(stderr_buf,driver.storage.bench.clock.errorReason); exit;
|
|
end;
|
|
bufWriteStr(stdout_buf,'Clock: '); bufWriteStrLn(stdout_buf,driver.storage.bench.clock.sourceName);
|
|
if mode<>bmManual then begin runSuite(device,disk,mib,mode=bmAuto,stdout_buf,stderr_buf); exit; end;
|
|
writeValue(stdout_buf,'Disk ',disk); writeValue(stdout_buf,', sector bytes=',device^.sectorSize);
|
|
writeValue(stdout_buf,', hardware depth=',device^.maxActive); bufWriteNewLine(stdout_buf);
|
|
bufWriteStrLn(stdout_buf,'Virtual/cached/sparse reads can exceed physical media speed. Device deltas include background I/O.');
|
|
runCase(device,mib,kib,depth,start,explicitStart,randomRead,stdout_buf,stderr_buf,nil);
|
|
end;
|
|
|
|
type
|
|
PBenchMock = ^TBenchMock;
|
|
TBenchMock = record
|
|
Pending : array[0..3] of PIORequest;
|
|
Count, Batch, Calls, FailLBA : uint32;
|
|
InvalidRange : boolean;
|
|
end;
|
|
procedure testDispatch(device : PStorage_Device; request : PIORequest);
|
|
var mock : PBenchMock; pending : PIORequest;
|
|
begin
|
|
mock := PBenchMock(device^.controllerId0); Inc(mock^.Calls);
|
|
if (request^.RequestType<>ioRead) or (request^.LBA>=device^.maxSectorCount) or
|
|
(request^.SectorCount>device^.maxSectorCount-request^.LBA) then mock^.InvalidRange := true;
|
|
mock^.Pending[mock^.Count] := request; Inc(mock^.Count);
|
|
if mock^.Count=mock^.Batch then while mock^.Count>0 do begin
|
|
Dec(mock^.Count); pending := mock^.Pending[mock^.Count];
|
|
if pending^.LBA=mock^.FailLBA then complete_io(pending,false,eIOError)
|
|
else complete_io(pending,true,eNone);
|
|
end;
|
|
end;
|
|
|
|
procedure UnitTest;
|
|
var run : PBench; device : PStorage_Device; slot : PSlot; value, i, passed, failed : uint32;
|
|
mock : PBenchMock; buffer : pointer; summary : PCaseResult;
|
|
procedure Assert(ok : boolean; name : pchar);
|
|
begin if ok then Inc(passed) else begin Inc(failed); io.syslog.logln('STORBENCH FAIL',name); end; end;
|
|
begin
|
|
passed := 0; failed := 0;
|
|
Assert(parseNumber('4294967295',$FFFFFFFF,value) and (value=$FFFFFFFF),'parse uint32 max');
|
|
Assert(not parseNumber('4294967296',$FFFFFFFF,value),'reject numeric overflow');
|
|
Assert(not parseNumber('-1',16384,value),'reject negative');
|
|
Assert(not parseNumber('',16384,value),'reject empty');
|
|
Assert(not parseNumber('1x',16384,value),'reject trailing junk');
|
|
Assert((commandMode(nil,0,value)=bmAuto) and (value=0),'no args select suite');
|
|
Assert((commandMode('2',2,value)=bmAuto) and (value=0),'disk and size select suite');
|
|
Assert((commandMode('auto',3,value)=bmAuto) and (value=1),'explicit auto arguments');
|
|
Assert((commandMode('run',7,value)=bmManual) and (value=1),'explicit manual arguments');
|
|
Assert((commandMode('2',4,value)=bmManual) and (value=0),'legacy detailed arguments');
|
|
Assert(commandMode('list',1,value)=bmList,'disk list mode');
|
|
driver.storage.bench.clock.UnitTest;
|
|
Assert(ticksToUS(1234567)=1234567,'hardware microseconds');
|
|
Assert(perSecond(1,1000000)=1,'hardware rate conversion');
|
|
Assert(perSecond(uint64(4096) shl 20,2000000)=(uint64(4096) shl 20) div 2,'rate beyond 4 GiB');
|
|
Assert(perSecond(100,0)=0,'zero ticks');
|
|
Assert((latencyBucket(0)=0) and (latencyBucket(1)=1) and (latencyBucket(3)=2) and (latencyBucket($FFFFFFFF)=31),'latency buckets');
|
|
summary := PCaseResult(kalloc(sizeof(TCaseResult)));
|
|
if summary<>nil then begin
|
|
Assert(suiteProfile(0,true,32,summary) and (summary^.KiB=1024) and (summary^.Depth=1) and not summary^.RandomRead,'auto sequential baseline');
|
|
Assert(suiteProfile(1,true,32,summary) and (summary^.Depth=16),'auto caps queued depth');
|
|
Assert(suiteProfile(1,true,1,summary) and (summary^.Depth=4),'auto tests software queue on serial devices');
|
|
Assert(suiteProfile(3,true,8,summary) and (summary^.KiB=4) and (summary^.Depth=1) and summary^.RandomRead,'auto random baseline');
|
|
Assert(suiteProfile(4,true,8,summary) and (summary^.KiB=4) and (summary^.Depth=8) and summary^.RandomRead,'auto random queued');
|
|
Assert(suiteProfile(5,true,8,summary) and (summary^.KiB=64) and summary^.RandomRead,'auto medium random');
|
|
Assert(not suiteProfile(6,true,8,summary),'auto has exactly six cases');
|
|
Assert(suiteProfile(11,false,1,summary) and (summary^.KiB=1024) and (summary^.Depth=32),'compare retains full matrix');
|
|
Assert(not suiteProfile(12,false,1,summary),'compare has exactly twelve cases');
|
|
kfree(void(summary));
|
|
end else Assert(false,'suite test allocation');
|
|
run := PBench(kalloc(sizeof(TBench))); device := PStorage_Device(kalloc(sizeof(TStorage_Device)));
|
|
if (run=nil) or (device=nil) then begin
|
|
if run<>nil then kfree(void(run)); if device<>nil then kfree(void(device));
|
|
io.syslog.logln('STORBENCH FAIL','test allocation'); exit;
|
|
end;
|
|
memset(uint32(run),0,sizeof(TBench)); memset(uint32(device),0,sizeof(TStorage_Device));
|
|
device^.dispatchRead := @testDispatch; device^.sectorSize := 512; device^.maxSectorCount := $FFFFFFFF;
|
|
Assert(configure(run,device,4096,1024,32,0,true,false) and (run^.Span=8388608),'4 GiB transfer does not wrap');
|
|
Assert(not configure(run,device,1,1024,33,0,true,false),'depth memory cap');
|
|
Assert(not configure(run,device,1,0,1,0,true,false),'zero request rejected');
|
|
Assert(not configure(run,device,1,4,1,$FFFFFFFF,true,false),'start beyond device rejected');
|
|
Assert(configure(run,device,1,4,1,$FFFFFFFE,true,false) and (run^.Span=1),'range clamp near uint32 end');
|
|
slot := @run^.Slots[0]; planRead(run,slot);
|
|
Assert((slot^.LBA=$FFFFFFFE) and (slot^.Sectors=1) and (run^.Remaining=0),'short final request');
|
|
device^.maxSectorCount := 4096;
|
|
Assert(configure(run,device,1,4,4,0,false,true) and (run^.StartLBA=2048),'default starting LBA');
|
|
for i := 1 to 256 do begin
|
|
planRead(run,slot);
|
|
Assert((slot^.LBA>=2048) and (slot^.LBA<=4096-slot^.Sectors),'random request stays in range');
|
|
end;
|
|
Assert(run^.Remaining=0,'random workload has exact sector count');
|
|
device^.sectorSize := 2048;
|
|
Assert(not configure(run,device,1,1,1,0,true,false),'request sector alignment');
|
|
run^.Completed := 20; run^.Histogram[0] := 18; run^.Histogram[2] := 1; run^.Histogram[10] := 1;
|
|
Assert(p95Ticks(run)=3,'95th percentile excludes slowest five percent');
|
|
{ Synthetic driver only: callbacks arrive inline in reverse groups of four. }
|
|
mock := PBenchMock(kalloc(sizeof(TBenchMock))); buffer := kalloc(4*65536);
|
|
device^.requestQueue := CFIFO_New(sizeof(pointer),8);
|
|
if (mock<>nil) and (buffer<>nil) and (device^.requestQueue<>nil) then begin
|
|
memset(uint32(mock),0,sizeof(TBenchMock)); mock^.Batch := 4; mock^.FailLBA := $FFFFFFFF;
|
|
device^.controllerId0 := uint32(mock); device^.sectorSize := 512; device^.maxActive := 4;
|
|
memset(uint32(run),0,sizeof(TBench)); run^.Buffer := buffer;
|
|
Assert(configure(run,device,1,64,4,0,true,false),'configure queued mock');
|
|
execute(run);
|
|
Assert((run^.Accepted=16) and (run^.Completed=16) and (run^.Bytes=1048576),'queued reads complete exact byte count');
|
|
Assert((run^.Peak=4) and (mock^.Count=0) and (run^.Active=0),'reverse groups drain at configured depth');
|
|
Assert(not mock^.InvalidRange,'benchmark issues only in-range reads');
|
|
memset(uint32(run),0,sizeof(TBench)); run^.Buffer := buffer; mock^.FailLBA := 128;
|
|
Assert(configure(run,device,1,64,4,0,true,false),'configure failing mock');
|
|
execute(run);
|
|
Assert((run^.Error=eIOError) and (run^.Accepted=4) and (run^.Completed=4),'failure stops new submissions and drains accepted requests');
|
|
Assert((run^.Bytes=3*65536) and (run^.Active=0),'failed reads excluded from throughput');
|
|
memset(uint32(run),0,sizeof(TBench)); run^.Buffer := buffer; mock^.FailLBA := $FFFFFFFF; mock^.Batch := 1;
|
|
Assert(configure(run,device,1,64,1,0,true,false),'configure immediate mock');
|
|
execute(run);
|
|
Assert((run^.Peak=1) and (run^.Bytes=1048576),'inline completions preserve peak and byte accounting');
|
|
memset(uint32(run),0,sizeof(TBench)); run^.Buffer := buffer;
|
|
Assert(configure(run,device,1,64,4,0,true,false),'configure removed device mock');
|
|
device^.removed := true; execute(run);
|
|
Assert((run^.Error=eDeviceRemoved) and (run^.Accepted=0) and (run^.Active=0),'submission rejection has no completion ownership');
|
|
end else Assert(false,'mock allocation');
|
|
if mock<>nil then kfree(void(mock)); if buffer<>nil then kfree(buffer);
|
|
if device^.requestQueue<>nil then CFIFO_Free(device^.requestQueue);
|
|
kfree(void(device)); kfree(void(run));
|
|
io.syslog.log('STORBENCH','Passed: '); io.syslog.writeint(passed);
|
|
io.syslog.writestring(' Failed: '); io.syslog.writeintln(failed);
|
|
end;
|
|
procedure init;
|
|
begin registerCommand('STORBENCH',@command,'Read-only disk benchmark. STORBENCH help for sizes, queue depths and comparison mode.'); end;
|
|
initialization
|
|
boot.mgr.registerBoot('driver.storage.bench',@init,'Storage Benchmark',BOOT_MGR_BARRIER_LATE);
|
|
end.
|