diff --git a/doc/src/driver/storage/driver.storage.test.md b/doc/src/driver/storage/driver.storage.test.md index dff0b55c..70b2652a 100644 --- a/doc/src/driver/storage/driver.storage.test.md +++ b/doc/src/driver/storage/driver.storage.test.md @@ -58,3 +58,111 @@ Registers the `STORTEST` and `DISKTEST` shell commands with `io.stdio`. Must be - `run_tests` exercises `makeAbsolutePathFrom`, `resolvePathFrom`, `changeDirectoryFrom`, `GetDirectoryListingFrom`, `newVirtualDirectory`, and `PathValid` including `.` and `..` traversal. - `run_disk_tests` is destructive and will irreversibly wipe the partition table of disk 0. It is never called automatically. - The `Assert` helper used in both procedures increments `passed` or `failed` counters and logs the test name on failure; passing tests produce no output to avoid noise. + + +## Raw read benchmark + +`STORBENCH` is implemented and registered by `driver.storage.bench`. It performs +read-only raw-device I/O; it does not invoke the destructive `DISKTEST` path. + +```text +STORBENCH [disk] [MiB] +STORBENCH list +STORBENCH run [disk] [MiB] [KiB/request] [depth] [seq|random] [start_lba] +STORBENCH compare [disk] [MiB] +STORBENCH help +``` + +Manual (`run`) defaults are disk 0, 128 MiB, 1024 KiB/request, sequential reads, and a depth of +`min(4, hardware depth)` (at least one). Examples: + +```text +STORBENCH 0 128 64 4 +STORBENCH 0 128 4 16 random +STORBENCH 0 256 1024 1 seq 2048 +STORBENCH compare 0 128 +``` + +The comparison runs sequential reads at 4, 64, and 1024 KiB/request, each at depths +1, 4, 16, and 32. The MiB argument applies to each of the twelve cases. Queue depth +may exceed hardware depth to expose software queuing and request merging. + +Limits are 1..16384 MiB, 1..1024 KiB/request, and depth 1..32. Request size must be +an exact number of device sectors. Buffers are allocated once per case, with a +separate slice for every outstanding request and a maximum total size of 32 MiB. +The transfer range is clamped to the device. Without an explicit LBA, reads start +at sector 2048 when the full requested range fits there; otherwise they start at +zero. Random mode uses a fixed xorshift seed and samples that range with replacement, +so total bytes read are bounded but unique coverage is not guaranteed. + +Reports include successful bytes, elapsed microseconds, MiB/s, IOPS, accepted and +completed request counts, peak outstanding requests, average/p95/max completion +latency, and per-run storage-counter deltas. Latencies are measured from submission +to callback. The p95 is an upper bound from logarithmic microsecond buckets, not an exact +microsecond percentile. Runs shorter than 10 ms are excluded from suite speed comparisons. +No console output occurs inside the measured loop. Any failure stops new work; +already accepted DMA is drained before buffers are freed, and failed requests +contribute no bytes or successful IOPS. + +Timing prefers the Hyper-V-compatible VM reference counter (100 ns ticks), detected +through its Hv#1 interface and reference-counter privilege. It needs no calibration. +Other platforms use an invariant TSC calibrated against two complete RTC seconds, +with up to three attempts to tolerate transient RTC updates or scheduling delays. +Calibration requires agreement within 1%; failures include the specific reason. +Unlike delivered timer interrupts, this clock advances while interrupts are masked. +Queue submission restores interrupts between requests. Filesystem caches are +bypassed, but drive and virtual-machine/host caches may still affect results. +Virtual or sparse disks can report throughput above physical-media speed. +Device counter deltas may include concurrent I/O from other processes; +benchmark-owned byte, request, and latency counters do not. + +`driver.storage.bench.UnitTest` runs memory-only parser, arithmetic, range, +queue-depth, completion-order, and error-accounting tests at boot. It never reads +or writes a real disk. The clock tests also verify that an advertised VM reference +counter advances with interrupts masked; they never calibrate the RTC at boot. + + +### Automatic suite and table output + +`STORBENCH` without arguments, or `STORBENCH auto [disk] [MiB]`, runs six +read-only profiles. Auto defaults to disk 0 and **32 MiB per profile** (up to +192 MiB total before device-size clamping). One or two numeric arguments select the auto suite: `STORBENCH 2 512` tests +disk 2 with 512 MiB per profile. `STORBENCH list` lists disk indices, capacities, +controllers, sector sizes, and hardware queue depths. Three or more numeric +arguments retain the legacy manual syntax; `run` selects manual mode explicitly. + +| Profile | Request size | Queue depth | +| --- | --- | --- | +| Seq large | 1024 KiB | 1 | +| Seq queued | 1024 KiB | queued | +| Seq medium | 64 KiB | queued | +| Random small | 4 KiB | 1 | +| Random queued | 4 KiB | queued | +| Random medium | 64 KiB | queued | + +Queued depth is the device hardware depth clamped to 4..16, allowing software +queue testing even on serial devices. A request smaller than a device sector is +increased to one sector. Each profile samples the same byte-span size, with the +existing fixed random seed and range clamping rules. + +Auto and compare modes print an aligned ASCII table with request KiB, queue +depth, MiB/s, IOPS, average latency, p95 latency upper bound, and status. Rows are +printed outside the measured loop. Successful runs shorter than 10 ms show `-` metrics and +`SHORT`, and are excluded from best-result and queue-gain summaries. Errors show +`FAILED`, stop the suite after accepted DMA has drained, and leave remaining +profiles unrun. The summary includes successful profiles, bytes actually read, +measured time, best sequential throughput, peak random IOPS, and (in auto mode) +queued/QD1 throughput ratios. These ratios describe the measured cases, including +any controller-cache effects; they are not a hardware rating. + +```text +STORBENCH +STORBENCH list +STORBENCH 2 512 +STORBENCH auto 0 64 +STORBENCH compare 0 32 +``` + +Profiles reuse the same raw device range. Host/drive caches are not flushed, so +later profiles may read warm data. These results are observed raw read throughput, +not a claim of uncached physical-media performance. diff --git a/src/arch/x86/isr/arch.x86.isr.tmr0.pas b/src/arch/x86/isr/arch.x86.isr.tmr0.pas index 0408693c..9dd86a7e 100644 --- a/src/arch/x86/isr/arch.x86.isr.tmr0.pas +++ b/src/arch/x86/isr/arch.x86.isr.tmr0.pas @@ -27,6 +27,10 @@ uses arch.x86.isr.mgr, arch.x86.idt; +const + PIT_INPUT_HZ = 1193182; + PIT_DIVISOR = 149; + procedure register(); procedure hook(hook_method : uint32); procedure unhook(hook_method : uint32); @@ -54,7 +58,7 @@ procedure register(); begin if not registered then begin asm - mov ax, 149 + mov ax, PIT_DIVISOR out $40, al mov al, ah out $40, al diff --git a/src/driver/storage/driver.storage.bench.clock.pas b/src/driver/storage/driver.storage.bench.clock.pas new file mode 100644 index 00000000..e631301c --- /dev/null +++ b/src/driver/storage/driver.storage.bench.clock.pas @@ -0,0 +1,284 @@ +{ Benchmark time must advance while IRQ delivery is masked. + Prefer the fixed-rate Hyper-V reference counter; RTC-calibrated TSC is a fallback. } +unit driver.storage.bench.clock; +interface +function prepare : boolean; +function ready : boolean; +function nowUS : uint64; +function sourceName : pchar; +function errorReason : pchar; +procedure UnitTest; +implementation +uses arch.x86.util, io.syslog; +var clockReady, clockBusy, referenceClock : boolean; + failure : pchar; + frequency, epoch : uint64; + +{ Hv#1 is an interface signature, not a vendor string: VirtualBox also + implements it. Never probe a synthetic MSR without its advertised privilege. + https://learn.microsoft.com/en-us/virtualization/hyper-v-on-windows/tlfs/timers } +function referenceAvailable(cpuECX, cpuEDX, maxLeaf, signature, privileges : uint32) : boolean; +begin + referenceAvailable := ((cpuECX and $80000000)<>0) and + ((cpuEDX and $20)<>0) and (maxLeaf >= $40000003) and + (signature=$31237648) and ((privileges and 2)<>0); +end; +function referenceSupported : boolean; +var cpuECX, cpuEDX, maxLeaf, signature, privileges : uint32; +begin + referenceSupported:=false; + asm + push ebx + mov eax,1 + cpuid + mov cpuECX,ecx + mov cpuEDX,edx + pop ebx + end; + if ((cpuECX and $80000000)=0) or ((cpuEDX and $20)=0) then exit; + asm + push ebx + mov eax,$40000000 + cpuid + mov maxLeaf,eax + mov eax,$40000001 + cpuid + mov signature,eax + pop ebx + end; + if (maxLeaf<$40000003) or (signature<>$31237648) then exit; + asm + push ebx + mov eax,$40000003 + cpuid + mov privileges,eax + pop ebx + end; + referenceSupported:=referenceAvailable(cpuECX,cpuEDX,maxLeaf,signature,privileges); +end; +function readReference : uint64; +var hi, lo : uint32; +begin + asm + mov ecx,$40000020 + rdmsr + mov lo,eax + mov hi,edx + end; + readReference:=(uint64(hi) shl 32) or lo; +end; +function referenceUS(value : uint64) : uint64; +begin referenceUS:=value div 10; end; { reference ticks are 100 ns } +function sourceName : pchar; +begin + if not clockReady then sourceName:='unavailable' + else if referenceClock then sourceName:='VM reference clock (no calibration)' + else sourceName:='RTC-calibrated invariant TSC'; +end; +function errorReason : pchar; +begin + if failure=nil then errorReason:='Clock initialization is already running; retry shortly.' + else errorReason:=failure; +end; + +function readCycles : uint64; +var hi, lo : uint32; +begin + { prepare checks SSE2 and invariant TSC before this is ever used. } + asm + lfence + rdtsc + mov lo,eax + mov hi,edx + lfence + end; + readCycles := (uint64(hi) shl 32) or lo; +end; +function supported : boolean; +var basic, extended, features, invariant : uint32; +begin + asm + push ebx + xor eax,eax + cpuid + mov basic,eax + mov eax,1 + cpuid + mov features,edx + mov eax,$80000000 + cpuid + mov extended,eax + pop ebx + end; + supported := false; + if (basic<1) or ((features and $04000010)<>$04000010) or (extended<$80000007) then exit; + asm + push ebx + mov eax,$80000007 + cpuid + mov invariant,edx + pop ebx + end; + supported := (invariant and $100)<>0; +end; +function toUS(cycles, hz : uint64) : uint64; +begin + toUS := 0; + if hz=0 then exit; + { Split before multiplying: raw TSC values can exceed 32 bits every second. } + toUS := (cycles div hz)*1000000 + ((cycles mod hz)*1000000) div hz; +end; +function acceptSamples(first, second : uint64; var hz : uint64) : boolean; +var delta : uint64; +begin + acceptSamples := false; hz := 0; + if (first<1000000) or (first>20000000000) or (second<1000000) or (second>20000000000) then exit; + if first>second then delta:=first-second else delta:=second-first; + if delta>first div 100 then exit; { reject >1% calibration disagreement } + hz := (first+second) div 2; acceptSamples := true; +end; +function rtcSecond(var seconds : uint32) : boolean; +var flags : uint32; before, after, raw, again, mode : uint8; +begin + { Keep each CMOS index/data transaction together, but never wait under CLI. } + asm pushfd; cli; pop flags end; + outb($70,$0A); before:=inb($71); + outb($70,$00); raw:=inb($71); + outb($70,$00); again:=inb($71); + outb($70,$0B); mode:=inb($71); + outb($70,$0A); after:=inb($71); + if (flags and $200)<>0 then asm sti end; + rtcSecond := false; + if (((before or after) and $80)<>0) or (raw<>again) then exit; + if (mode and 4)<>0 then seconds:=raw + else begin + if ((raw and $0F)>9) or ((raw shr 4)>5) then exit; + seconds:=(raw shr 4)*10+(raw and $0F); + end; + rtcSecond := seconds<60; +end; +function nextSecond(var previous : uint32; var stamp : uint64) : boolean; +var i, current : uint32; start, now : uint64; +begin + nextSecond:=false; start:=readCycles; + for i:=1 to 2000000 do begin + if rtcSecond(current) then if current<>previous then begin + if current<>(previous+1) mod 60 then exit; + previous:=current; stamp:=readCycles; nextSecond:=true; exit; + end; + now:=readCycles; + if (now40000000000) then exit; + end; +end; +function prepare : boolean; +var flags, second, attempt, i : uint32; + first, middle, last, hz : uint64; ok : boolean; +begin + prepare:=false; + asm pushfd; cli; pop flags end; + if clockReady then begin + if (flags and $200)<>0 then asm sti end; + prepare:=true; exit; + end; + if clockBusy then begin + if (flags and $200)<>0 then asm sti end; + exit; + end; + clockBusy:=true; failure:=nil; + if (flags and $200)<>0 then asm sti end; + ok:=false; + if referenceSupported then begin + first:=readReference; + for i:=1 to 10000 do begin + last:=readReference; + if last=10 then begin ok:=true; break; end; + end; + if ok then begin + asm cli end; + referenceClock:=true; epoch:=first; clockReady:=true; clockBusy:=false; + if (flags and $200)<>0 then asm sti end; + io.syslog.logln('BENCHCLOCK','Using VM reference clock; no calibration.'); + prepare:=true; exit; + end; + failure:='VM reference clock did not advance.'; + end; + if (flags and $200)=0 then failure:='TSC fallback needs interrupts enabled.' + else if not supported then failure:='No VM reference clock or invariant TSC available.' + else begin + { A transient RTC update or a delayed VM timeslice must not permanently + reject an otherwise usable clock. Retry bounded full-second samples. } + for attempt:=1 to 3 do begin + ok:=false; + for i:=1 to 10000 do if rtcSecond(second) then begin ok:=true; break; end; + if ok then ok:=nextSecond(second,first); + if ok then ok:=nextSecond(second,middle); + if ok then ok:=nextSecond(second,last); + if ok then begin + ok:=(middle>first) and (last>middle); + if ok then ok:=acceptSamples(middle-first,last-middle,hz); + end; + if ok then break; + end; + if not ok then failure:='RTC/TSC fallback samples were unstable; retry when the system is idle.'; + end; + asm cli end; + if ok then begin frequency:=hz; epoch:=last; referenceClock:=false; clockReady:=true; end; + clockBusy:=false; + if (flags and $200)<>0 then asm sti end; + prepare:=ok; +end; +function ready : boolean; +begin ready:=clockReady; end; +function nowUS : uint64; +var value : uint64; +begin + nowUS:=0; + if not clockReady then exit; + if referenceClock then begin + value:=readReference; + if value>=epoch then nowUS:=referenceUS(value-epoch); + end else begin + value:=readCycles; + if value>=epoch then nowUS:=toUS(value-epoch,frequency); + end; +end; +procedure UnitTest; +var hz, first, last : uint64; passed, failed, flags, i : uint32; + procedure Assert(ok : boolean; name : pchar); + begin if ok then Inc(passed) else begin Inc(failed); io.syslog.logln('BENCHCLOCK FAIL',name); end; end; +begin + passed:=0;failed:=0; + Assert(referenceAvailable($80000000,$20,$40000006,$31237648,$AF2),'VBox Hv#1 reference clock'); + Assert(not referenceAvailable(0,$20,$40000006,$31237648,$AF2),'no hypervisor no MSR'); + Assert(not referenceAvailable($80000000,0,$40000006,$31237648,$AF2),'no RDMSR support'); + Assert(not referenceAvailable($80000000,$20,$40000002,$31237648,$AF2),'missing feature leaf'); + Assert(not referenceAvailable($80000000,$20,$40000006,0,$AF2),'different hypervisor interface'); + Assert(not referenceAvailable($80000000,$20,$40000006,$31237648,$AF0),'missing reference privilege'); + Assert(referenceUS(10000000)=1000000,'100 ns ticks to microseconds'); + Assert(referenceUS(50000000000)=5000000000,'reference conversion beyond uint32'); + Assert(referenceUS(9)=0,'reference sub-microsecond truncation'); + { Safe hardware smoke test only when CPUID explicitly grants MSR access. + No disk I/O and no RTC calibration occurs during boot tests. } + if referenceSupported then begin + Assert(prepare,'VM reference clock initialization'); + if ready then begin + asm pushfd; cli; pop flags end; + first:=nowUS; last:=first; + for i:=1 to 10000 do begin last:=nowUS; if last>first then break; end; + if (flags and $200)<>0 then asm sti end; + Assert(last>first,'VM clock advances with interrupts masked'); + end; + end; + Assert(acceptSamples(4291845105,4291845105,hz) and (hz=4291845105),'near uint32 frequency boundary'); + Assert(acceptSamples(5000000000,5000000000,hz) and (hz=5000000000),'frequency above uint32'); + Assert(not acceptSamples(5000000000,5100000000,hz),'reject unstable calibration'); + Assert(not acceptSamples(0,0,hz),'reject missing clock'); + Assert(toUS(10000000000,5000000000)=2000000,'64-bit elapsed conversion'); + Assert(toUS(5000002500,5000000000)=1000000,'fractional microsecond truncation'); + Assert(toUS(18000000000000,5000000000)=3600000000,'one hour does not overflow'); + Assert(toUS(123,0)=0,'invalid divisor'); + io.syslog.log('BENCHCLOCK','Passed: '); io.syslog.writeint(passed); + io.syslog.writestring(' Failed: '); io.syslog.writeintln(failed); +end; +end. diff --git a/src/driver/storage/driver.storage.bench.pas b/src/driver/storage/driver.storage.bench.pas new file mode 100644 index 00000000..6406cc05 --- /dev/null +++ b/src/driver/storage/driver.storage.bench.pas @@ -0,0 +1,634 @@ +{ Read-only raw storage benchmark. No disk I/O is performed by UnitTest. } +unit driver.storage.bench; +interface +procedure init; +procedure UnitTest; +implementation +uses boot.mgr, io.stdio, io.syslog, core.strings, core.util, arch.x86.util, + arch.x86.bda, arch.x86.isr.tmr0, memory.heap, proc.mgr, proc.types, + driver.storage.types, driver.storage.mgr, core.ds.cfifo, driver.storage.bench.clock; +const + MAX_DEPTH = 32; + DEFAULT_MIB = 128; + AUTO_MIB = 32; + MIN_SAMPLE_US = 10000; { avoid presenting millisecond cache hits as drive ratings } +type + PCaseResult = ^TCaseResult; + TCaseResult = record + Name : pchar; + KiB, Depth : uint32; + RandomRead, Measured : boolean; + Error : TError; + Rate10, IOPS, AvgUS, P95US, Bytes : uint64; + Elapsed : uint64; + end; + PBench = ^TBench; + PSlot = ^TSlot; + TSlot = record + Owner : PBench; + Buffer : puint32; + State, LBA, Sectors : uint32; { State: 0 free, 1 DMA, 2 completed } + Started : uint64; + end; + TBench = record + Device : PStorage_Device; + Process : PProcessContext; + Buffer : pointer; + Depth, RequestBytes, ChunkSectors, StartLBA, Span, Remaining, Seed : uint32; + RandomRead : boolean; + Active, Peak, WakeSequence, Accepted, Completed, Successful : uint32; + Error : TError; + ErrorLBA : uint32; + StartTick, Elapsed, LatencyMax : uint64; + Bytes, LatencyTotal : uint64; + Histogram : array[0..31] of uint32; + BeforeStats, AfterStats : TStorageStats; + Slots : array[0..MAX_DEPTH-1] of TSlot; + end; + +function nowTick : uint64; +begin nowTick := driver.storage.bench.clock.nowUS; end; + +{ Reject overflow before multiplication; CLI parsing must never wrap. } +function parseNumber(text : pchar; limit : uint32; var value : uint32) : boolean; +var i, digit : uint32; +begin + parseNumber := false; value := 0; + if (text=nil) or (text[0]=#0) then exit; + i := 0; + while text[i]<>#0 do begin + if (text[i]<'0') or (text[i]>'9') then exit; + digit := ord(text[i])-ord('0'); + if (value>limit div 10) or ((value=limit div 10) and (digit>limit mod 10)) then exit; + value := value*10+digit; Inc(i); + end; + parseNumber := true; +end; + +function ticksToUS(ticks : uint64) : uint64; +begin ticksToUS := ticks; end; { samples now use independent hardware microseconds } +function perSecond(value, ticks : uint64) : uint64; +begin + perSecond := 0; + if ticks<>0 then perSecond := (value*1000000) div ticks; +end; +function latencyBucket(ticks : uint64) : uint32; +var bucket : uint32; +begin + bucket := 0; + while (ticks<>0) and (bucket<31) do begin Inc(bucket); ticks := ticks shr 1; end; + latencyBucket := bucket; +end; +function p95Ticks(run : PBench) : uint64; +var target, count, i : uint32; +begin + p95Ticks := 0; + if run^.Completed=0 then exit; + target := run^.Completed - run^.Completed div 20; count := 0; + for i := 0 to 31 do begin + Inc(count,run^.Histogram[i]); + if count>=target then begin + if i=31 then p95Ticks := run^.LatencyMax + else p95Ticks := (uint32(1) shl i)-1; + exit; + end; + end; +end; + +procedure writeNumber(output : POutBuf; value : uint64); +var digits : array[0..20] of char; n : uint32; +begin + n := 0; + repeat digits[n] := char(ord('0')+value mod 10); value := value div 10; Inc(n); until value=0; + while n>0 do begin Dec(n); bufWriteChar(output,digits[n]); end; +end; +procedure writeValue(output : POutBuf; name : pchar; value : uint64); +begin bufWriteStr(output,name); writeNumber(output,value); end; + +{ All geometry arithmetic is bounded before buffer allocation or I/O. } +function configure(run : PBench; device : PStorage_Device; mib, kib, depth, start : uint32; + explicitStart, randomRead : boolean) : boolean; +var sectors : uint64; +begin + configure := false; + if device=nil then exit; + if device^.removed or (device^.dispatchRead=nil) or (device^.maxSectorCount=0) then exit; + if (device^.sectorSize=0) or (device^.sectorSize>1048576) then exit; + if (mib=0) or (mib>16384) or (kib=0) or (kib>1024) or (depth=0) or (depth>MAX_DEPTH) then exit; + if ((kib*1024) mod device^.sectorSize)<>0 then exit; + sectors := (uint64(mib) shl 20) div device^.sectorSize; + if not explicitStart then begin + start := 0; + if device^.maxSectorCount>2048 then + if sectors<=device^.maxSectorCount-2048 then start := 2048; + end; + if start>=device^.maxSectorCount then exit; + if sectors>device^.maxSectorCount-start then sectors := device^.maxSectorCount-start; + if sectors=0 then exit; + run^.Device := device; run^.Depth := depth; run^.RequestBytes := kib*1024; + run^.ChunkSectors := run^.RequestBytes div device^.sectorSize; + run^.StartLBA := start; run^.Span := uint32(sectors); run^.Remaining := run^.Span; + run^.Seed := $6D2B79F5; run^.RandomRead := randomRead; + configure := true; +end; + +procedure planRead(run : PBench; slot : PSlot); +var span : uint32; +begin + slot^.Sectors := run^.ChunkSectors; + if slot^.Sectors>run^.Remaining then slot^.Sectors := run^.Remaining; + if run^.RandomRead then begin + run^.Seed := run^.Seed xor (run^.Seed shl 13); + run^.Seed := run^.Seed xor (run^.Seed shr 17); + run^.Seed := run^.Seed xor (run^.Seed shl 5); + span := run^.Span-slot^.Sectors+1; + slot^.LBA := run^.StartLBA + run^.Seed mod span; + end else slot^.LBA := run^.StartLBA + (run^.Span-run^.Remaining); + Dec(run^.Remaining,slot^.Sectors); +end; + +procedure readDone(error : TError; data : pointer); +var slot : PSlot; run : PBench; ticks : uint64; +begin + slot := PSlot(data); run := slot^.Owner; + ticks := nowTick-slot^.Started; + Inc(run^.Completed); Inc(run^.LatencyTotal,ticks); + if ticks>run^.LatencyMax then run^.LatencyMax := ticks; + Inc(run^.Histogram[latencyBucket(ticks)]); + if error=eNone then begin + Inc(run^.Successful); Inc(run^.Bytes,uint64(slot^.Sectors)*run^.Device^.sectorSize); + end else if run^.Error=eNone then begin run^.Error := error; run^.ErrorLBA := slot^.LBA; end; + Dec(run^.Active); slot^.State := 2; + if run^.Process<>nil then run^.Process^.State := psReady; + Inc(run^.WakeSequence); +end; + +{ Task context only. Reserve before submit to support immediate callbacks. + Stop submitting on the first error, but drain accepted DMA before freeing. } +procedure execute(run : PBench); +var i, flags, wake, attemptedActive : uint32; slot : PSlot; err : TError; finished, ready : boolean; +begin + run^.Process := CurrentProcess; + for i := 0 to run^.Depth-1 do begin + run^.Slots[i].Owner := run; + run^.Slots[i].Buffer := puint32(uint32(run^.Buffer)+i*run^.RequestBytes); + end; + storage_get_stats(run^.Device,run^.BeforeStats); run^.StartTick := nowTick; + repeat + for i := 0 to run^.Depth-1 do begin + asm pushfd; cli; pop flags end; + slot := @run^.Slots[i]; + if slot^.State=2 then slot^.State := 0; + if (slot^.State=0) and (run^.Remaining>0) and (run^.Error=eNone) then begin + planRead(run,slot); slot^.State := 1; slot^.Started := nowTick; + Inc(run^.Active); Inc(run^.Accepted); attemptedActive := run^.Active; + err := storage_read_async(run^.Device,slot^.LBA,slot^.Sectors,slot^.Buffer,@readDone,slot); + if err<>eNone then begin + Dec(run^.Active); Dec(run^.Accepted); slot^.State := 0; + if run^.Error=eNone then begin run^.Error := err; run^.ErrorLBA := slot^.LBA; end; + end; + if (err=eNone) and (attemptedActive>run^.Peak) then run^.Peak := attemptedActive; + end; + if (flags and $200)<>0 then asm sti end; + end; + asm pushfd; cli; pop flags end; + finished := (run^.Active=0) and ((run^.Remaining=0) or (run^.Error<>eNone)); + ready := false; + for i := 0 to run^.Depth-1 do if run^.Slots[i].State=2 then ready := true; + wake := run^.WakeSequence; + if not finished and not ready and (run^.Process<>nil) then run^.Process^.State := psAwaiting; + if (flags and $200)<>0 then asm sti end; + if not finished and not ready then + while puint32(@run^.WakeSequence)^=wake do asm hlt end; + until finished; + run^.Elapsed := nowTick-run^.StartTick; + storage_get_stats(run^.Device,run^.AfterStats); +end; + +procedure report(run : PBench; output, errors : POutBuf); +var rate, ticks, commands : uint64; +begin + if run^.Error<>eNone then begin + writeValue(errors,'Read failed at LBA ',run^.ErrorLBA); + writeValue(errors,', error=',ord(run^.Error)); + bufWriteStrLn(errors,'. All accepted reads drained; results below are partial.'); + end; + writeValue(output,' Read ',run^.Bytes div 1024); bufWriteStr(output,' KiB'); + writeValue(output,' in ',ticksToUS(run^.Elapsed)); bufWriteStr(output,' us; '); + if run^.Elapsed0 then begin + writeValue(output,' Latency avg: ',ticksToUS(run^.LatencyTotal) div run^.Completed); + ticks := p95Ticks(run); + if ticks=0 then writeValue(output,' us; p95 < ',ticksToUS(1)+1) + else writeValue(output,' us; p95 < ',ticksToUS(ticks+1)+1); + writeValue(output,' us; max < ',ticksToUS(uint64(run^.LatencyMax)+1)+1); + bufWriteStrLn(output,' us (hardware clock; p95 uses log buckets)'); + end; + commands := uint32(run^.AfterStats.Commands-run^.BeforeStats.Commands); + writeValue(output,' Device deltas: commands=',commands); + writeValue(output,', merged=',uint32(run^.AfterStats.Merged-run^.BeforeStats.Merged)); + writeValue(output,', failed=',uint32(run^.AfterStats.Failed-run^.BeforeStats.Failed)); + writeValue(output,', queue-full=',uint32(run^.AfterStats.QueueFull-run^.BeforeStats.QueueFull)); + bufWriteNewLine(output); + +end; + +function runCase(device : PStorage_Device; mib, kib, depth, start : uint32; + explicitStart, randomRead : boolean; output, errors : POutBuf; + summary : PCaseResult) : boolean; +var run : PBench; +begin + runCase := false; + run := PBench(kalloc(sizeof(TBench))); + if summary<>nil then summary^.Error := eOutOfMemory; + if run=nil then begin bufWriteStrLn(errors,'Out of memory allocating benchmark state.'); exit; end; + memset(uint32(run),0,sizeof(TBench)); + if not configure(run,device,mib,kib,depth,start,explicitStart,randomRead) then begin + if summary<>nil then summary^.Error := eInvalidArgument; + bufWriteStrLn(errors,'Invalid/unreadable device, range, or request size (must be a whole number of device sectors).'); + kfree(void(run)); exit; + end; + run^.Buffer := kalloc(run^.Depth*run^.RequestBytes); + if run^.Buffer=nil then begin bufWriteStrLn(errors,'Out of memory allocating benchmark buffers.'); kfree(void(run)); exit; end; + memset(uint32(run^.Buffer),0,run^.Depth*run^.RequestBytes); + if summary=nil then begin + if randomRead then bufWriteStr(output,'Random') else bufWriteStr(output,'Sequential'); + writeValue(output,' read: ',kib); writeValue(output,' KiB/request, depth=',depth); + writeValue(output,', LBA=',run^.StartLBA); + writeValue(output,', span=',uint64(run^.Span)*device^.sectorSize div 1024); bufWriteStrLn(output,' KiB'); + end; + execute(run); runCase := run^.Error=eNone; + if summary=nil then report(run,output,errors) + else begin + summary^.Error := run^.Error; summary^.Bytes := run^.Bytes; + summary^.Elapsed := run^.Elapsed; summary^.Measured := run^.Elapsed>=MIN_SAMPLE_US; + if summary^.Measured then begin + summary^.Rate10 := perSecond(run^.Bytes*10,run^.Elapsed) div 1048576; + summary^.IOPS := perSecond(run^.Successful,run^.Elapsed); + end; + if run^.Completed>0 then begin + summary^.AvgUS := ticksToUS(run^.LatencyTotal) div run^.Completed; + summary^.P95US := ticksToUS(uint64(p95Ticks(run))+1)+1; + end; + if run^.Error<>eNone then begin + writeValue(errors,'Read failed at LBA ',run^.ErrorLBA); + writeValue(errors,', error=',ord(run^.Error)); bufWriteStrLn(errors,'. Accepted reads drained.'); + end; + end; + kfree(run^.Buffer); kfree(void(run)); +end; + +{ Fixed-width ASCII output stays readable in terminals and redirected logs. } +procedure textCell(output : POutBuf; text : pchar; width : uint32); +var i : uint32; +begin + i := 0; + while (i#0) do begin bufWriteChar(output,text[i]); Inc(i); end; + while ieNone then bufWriteStrLn(output,' FAILED') else bufWriteStrLn(output,' SHORT'); + end; +end; +function suiteProfile(index : uint32; automatic : boolean; hardwareDepth : uint32; + summary : PCaseResult) : boolean; +var queued : uint32; +begin + suiteProfile := false; + memset(uint32(summary),0,sizeof(TCaseResult)); + queued := hardwareDepth; if queued<4 then queued:=4; if queued>16 then queued:=16; + if automatic then begin + if index>=6 then exit; + case index of + 0:begin summary^.Name:='Seq large'; summary^.KiB:=1024; summary^.Depth:=1; end; + 1:begin summary^.Name:='Seq queued'; summary^.KiB:=1024; summary^.Depth:=queued; end; + 2:begin summary^.Name:='Seq medium'; summary^.KiB:=64; summary^.Depth:=queued; end; + 3:begin summary^.Name:='Random small'; summary^.KiB:=4; summary^.Depth:=1; end; + 4:begin summary^.Name:='Random queued'; summary^.KiB:=4; summary^.Depth:=queued; end; + 5:begin summary^.Name:='Random medium'; summary^.KiB:=64; summary^.Depth:=queued; end; + end; + summary^.RandomRead := index>=3; + end else begin + if index>=12 then exit; + summary^.Name := 'Sequential'; + case index div 4 of 0:summary^.KiB:=4; 1:summary^.KiB:=64; 2:summary^.KiB:=1024; end; + case index mod 4 of 0:summary^.Depth:=1; 1:summary^.Depth:=4; 2:summary^.Depth:=16; 3:summary^.Depth:=32; end; + end; + suiteProfile := true; +end; +procedure queueGain(output : POutBuf; name : pchar; baseline, queued : PCaseResult); +begin + if (baseline^.Error<>eNone) or (queued^.Error<>eNone) or + not baseline^.Measured or not queued^.Measured or (baseline^.Rate10=0) then exit; + bufWriteStr(output,name); numberCell(output,(queued^.Rate10*10) div baseline^.Rate10,0,true); + bufWriteStrLn(output,'x QD1 throughput'); +end; +procedure runSuite(device : PStorage_Device; disk, mib : uint32; automatic : boolean; + output, errors : POutBuf); +var summaries, row : PCaseResult; count, i, completed, bestSeq, bestRandom : uint32; + totalBytes, totalTicks : uint64; ok : boolean; +begin + if automatic then count:=6 else count:=12; + summaries := PCaseResult(kalloc(count*sizeof(TCaseResult))); + if summaries=nil then begin bufWriteStrLn(errors,'Out of memory allocating results.'); exit; end; + memset(uint32(summaries),0,count*sizeof(TCaseResult)); + bufWriteNewLine(output); tableLine(output); + if automatic then bufWriteStrLn(output,' STORAGE BENCHMARK / AUTO') else bufWriteStrLn(output,' STORAGE BENCHMARK / COMPARE'); + writeValue(output,' Disk ',disk); writeValue(output,' | ',mib); writeValue(output,' MiB/profile | ',count); + writeValue(output,' profiles | hardware QD ',device^.maxActive); bufWriteNewLine(output); + bufWriteStrLn(output,' Raw read throughput (includes host/drive cache effects).'); + bufWriteStrLn(output,' Asuro filesystem cache bypassed. Host/drive caches are not flushed.'); + bufWriteStrLn(output,' Profiles reuse the same range; later reads may be warm-cache hits.'); + writeValue(output,' Device capacity: ',(uint64(device^.maxSectorCount)*device^.sectorSize) div 1048576); bufWriteStrLn(output,' MiB'); + if (uint64(mib) shl 20)>uint64(device^.maxSectorCount)*device^.sectorSize then + bufWriteStrLn(output,' Requested size exceeds this disk; each profile is clamped to capacity.'); + tableHeader(output); + completed:=0; bestSeq:=$FFFFFFFF; bestRandom:=$FFFFFFFF; totalBytes:=0; totalTicks:=0; + for i:=0 to count-1 do begin + row:=@summaries[i]; + if not suiteProfile(i,automatic,device^.maxActive,row) then break; + { Keep the small-request profiles usable on larger-sector devices. } + if (row^.KiB*1024summaries[bestSeq].Rate10 then bestSeq:=i; + end else begin + if bestRandom=$FFFFFFFF then bestRandom:=i + else if row^.IOPS>summaries[bestRandom].IOPS then bestRandom:=i; + end; + end; + end; + tableLine(output); + if bestSeq<>$FFFFFFFF then begin + bufWriteStr(output,' Best sequential: '); numberCell(output,summaries[bestSeq].Rate10,0,true); + writeValue(output,' MiB/s (',summaries[bestSeq].KiB); writeValue(output,' KiB, QD ',summaries[bestSeq].Depth); bufWriteStrLn(output,')'); + end; + if bestRandom<>$FFFFFFFF then begin + writeValue(output,' Peak random: ',summaries[bestRandom].IOPS); bufWriteStr(output,' IOPS ('); + writeNumber(output,summaries[bestRandom].KiB); writeValue(output,' KiB, QD ',summaries[bestRandom].Depth); bufWriteStrLn(output,')'); + end; + if automatic then begin + if completed>=2 then queueGain(output,' Sequential queue gain: ',@summaries[0],@summaries[1]); + if completed>=5 then queueGain(output,' Small random queue gain: ',@summaries[3],@summaries[4]); + end; + writeValue(output,' Finished ',completed); writeValue(output,'/',count); + writeValue(output,' profiles | read ',totalBytes div 1024); + writeValue(output,' KiB | measured ',ticksToUS(totalTicks) div 1000); bufWriteStrLn(output,' ms'); + if completed2 then commandMode:=bmManual; { retain detailed numeric syntax } +end; +procedure usage(output : POutBuf); +begin + bufWriteStrLn(output,'STORBENCH [disk] [MiB] Default six-profile suite'); + bufWriteStrLn(output,'STORBENCH auto [disk] [MiB] Same suite (default 32 MiB/profile)'); + bufWriteStrLn(output,'STORBENCH list List disks and capacities'); + bufWriteStrLn(output,'STORBENCH compare [disk] [MiB] Twelve sequential cases'); + bufWriteStrLn(output,'STORBENCH run [disk] [MiB] [KiB/request] [depth] [seq|random] [start_lba]'); + bufWriteStrLn(output,'Examples: STORBENCH 2 512 STORBENCH run 2 512 64 4'); + bufWriteStrLn(output,'Manual defaults: disk 0, 128 MiB, 1024 KiB/request, depth up to 4, sequential.'); + bufWriteStrLn(output,'Limits: 1..16384 MiB, 1..1024 KiB/request, depth 1..32; sizes clamp to disk.'); + bufWriteStrLn(output,'Read-only. Cached/sparse VM reads do not measure physical media speed.'); +end; +procedure listDisks(output : POutBuf); +var count, i : uint32; device : PStorage_Device; +begin + count:=get_device_count; + if count=0 then begin bufWriteStrLn(output,'No storage devices.'); exit; end; + bufWriteStrLn(output,'Disk Controller MiB Sector bytes HW QD'); + tableLine(output); + for i:=0 to count-1 do begin + device:=get_device(i); + if device<>nil then begin + numberCell(output,i,4,false); bufWriteStr(output,' '); + textCell(output,controller_type_2_string(device^.controller),16); + numberCell(output,(uint64(device^.maxSectorCount)*device^.sectorSize) div 1048576,9,false); + numberCell(output,device^.sectorSize,17,false); numberCell(output,device^.maxActive,7,false); + if device^.removed then bufWriteStr(output,' removed'); + bufWriteNewLine(output); + end; + end; + bufWriteStrLn(output,'Select a disk and MiB per profile: STORBENCH 2 512'); +end; +procedure command(params : PParamList; stdin_buf, stdout_buf, stderr_buf : POutBuf); +var count, disk, mib, kib, depth, start, base : uint32; + mode : TBenchMode; randomRead, explicitStart : boolean; device : PStorage_Device; first : pchar; +begin + count:=paramCount(params); first:=nil; + if count>0 then first:=getParam(0,params); + mode:=commandMode(first,count,base); + if mode=bmHelp then begin usage(stdout_buf); exit; end; + if mode=bmList then begin + if count<>1 then usage(stderr_buf) else listDisks(stdout_buf); + exit; + end; + if ((mode<>bmManual) and (count>base+2)) or ((mode=bmManual) and (count>base+6)) then begin usage(stderr_buf); exit; end; + disk:=0; mib:=DEFAULT_MIB; kib:=1024; start:=0; + if mode=bmAuto then mib:=AUTO_MIB; + randomRead:=false; explicitStart:=false; + if count>base then if not parseNumber(getParam(base,params),$FFFFFFFF,disk) then begin usage(stderr_buf); exit; end; + if count>base+1 then if not parseNumber(getParam(base+1,params),16384,mib) or (mib=0) then begin usage(stderr_buf); exit; end; + device:=get_device(disk); + if device=nil then begin bufWriteStrLn(stderr_buf,'Storage device not found. Use STORBENCH list.'); exit; end; + depth:=device^.maxActive; if depth>4 then depth:=4; if depth=0 then depth:=1; + if mode=bmManual then begin + if count>base+2 then if not parseNumber(getParam(base+2,params),1024,kib) or (kib=0) then begin usage(stderr_buf); exit; end; + if count>base+3 then if not parseNumber(getParam(base+3,params),MAX_DEPTH,depth) or (depth=0) then begin usage(stderr_buf); exit; end; + if count>base+4 then begin + randomRead:=stringEqualsCI(getParam(base+4,params),'random'); + if not randomRead and not stringEqualsCI(getParam(base+4,params),'seq') then begin usage(stderr_buf); exit; end; + end; + if count>base+5 then begin + explicitStart:=true; + if not parseNumber(getParam(base+5,params),$FFFFFFFF,start) then begin usage(stderr_buf); exit; end; + end; + end; + if not driver.storage.bench.clock.prepare then begin + bufWriteStr(stderr_buf,'Benchmark clock unavailable: '); + bufWriteStrLn(stderr_buf,driver.storage.bench.clock.errorReason); exit; + end; + bufWriteStr(stdout_buf,'Clock: '); bufWriteStrLn(stdout_buf,driver.storage.bench.clock.sourceName); + if mode<>bmManual then begin runSuite(device,disk,mib,mode=bmAuto,stdout_buf,stderr_buf); exit; end; + writeValue(stdout_buf,'Disk ',disk); writeValue(stdout_buf,', sector bytes=',device^.sectorSize); + writeValue(stdout_buf,', hardware depth=',device^.maxActive); bufWriteNewLine(stdout_buf); + bufWriteStrLn(stdout_buf,'Virtual/cached/sparse reads can exceed physical media speed. Device deltas include background I/O.'); + runCase(device,mib,kib,depth,start,explicitStart,randomRead,stdout_buf,stderr_buf,nil); +end; + +type + PBenchMock = ^TBenchMock; + TBenchMock = record + Pending : array[0..3] of PIORequest; + Count, Batch, Calls, FailLBA : uint32; + InvalidRange : boolean; + end; +procedure testDispatch(device : PStorage_Device; request : PIORequest); +var mock : PBenchMock; pending : PIORequest; +begin + mock := PBenchMock(device^.controllerId0); Inc(mock^.Calls); + if (request^.RequestType<>ioRead) or (request^.LBA>=device^.maxSectorCount) or + (request^.SectorCount>device^.maxSectorCount-request^.LBA) then mock^.InvalidRange := true; + mock^.Pending[mock^.Count] := request; Inc(mock^.Count); + if mock^.Count=mock^.Batch then while mock^.Count>0 do begin + Dec(mock^.Count); pending := mock^.Pending[mock^.Count]; + if pending^.LBA=mock^.FailLBA then complete_io(pending,false,eIOError) + else complete_io(pending,true,eNone); + end; +end; + +procedure UnitTest; +var run : PBench; device : PStorage_Device; slot : PSlot; value, i, passed, failed : uint32; + mock : PBenchMock; buffer : pointer; summary : PCaseResult; + procedure Assert(ok : boolean; name : pchar); + begin if ok then Inc(passed) else begin Inc(failed); io.syslog.logln('STORBENCH FAIL',name); end; end; +begin + passed := 0; failed := 0; + Assert(parseNumber('4294967295',$FFFFFFFF,value) and (value=$FFFFFFFF),'parse uint32 max'); + Assert(not parseNumber('4294967296',$FFFFFFFF,value),'reject numeric overflow'); + Assert(not parseNumber('-1',16384,value),'reject negative'); + Assert(not parseNumber('',16384,value),'reject empty'); + Assert(not parseNumber('1x',16384,value),'reject trailing junk'); + Assert((commandMode(nil,0,value)=bmAuto) and (value=0),'no args select suite'); + Assert((commandMode('2',2,value)=bmAuto) and (value=0),'disk and size select suite'); + Assert((commandMode('auto',3,value)=bmAuto) and (value=1),'explicit auto arguments'); + Assert((commandMode('run',7,value)=bmManual) and (value=1),'explicit manual arguments'); + Assert((commandMode('2',4,value)=bmManual) and (value=0),'legacy detailed arguments'); + Assert(commandMode('list',1,value)=bmList,'disk list mode'); + driver.storage.bench.clock.UnitTest; + Assert(ticksToUS(1234567)=1234567,'hardware microseconds'); + Assert(perSecond(1,1000000)=1,'hardware rate conversion'); + Assert(perSecond(uint64(4096) shl 20,2000000)=(uint64(4096) shl 20) div 2,'rate beyond 4 GiB'); + Assert(perSecond(100,0)=0,'zero ticks'); + Assert((latencyBucket(0)=0) and (latencyBucket(1)=1) and (latencyBucket(3)=2) and (latencyBucket($FFFFFFFF)=31),'latency buckets'); + summary := PCaseResult(kalloc(sizeof(TCaseResult))); + if summary<>nil then begin + Assert(suiteProfile(0,true,32,summary) and (summary^.KiB=1024) and (summary^.Depth=1) and not summary^.RandomRead,'auto sequential baseline'); + Assert(suiteProfile(1,true,32,summary) and (summary^.Depth=16),'auto caps queued depth'); + Assert(suiteProfile(1,true,1,summary) and (summary^.Depth=4),'auto tests software queue on serial devices'); + Assert(suiteProfile(3,true,8,summary) and (summary^.KiB=4) and (summary^.Depth=1) and summary^.RandomRead,'auto random baseline'); + Assert(suiteProfile(4,true,8,summary) and (summary^.KiB=4) and (summary^.Depth=8) and summary^.RandomRead,'auto random queued'); + Assert(suiteProfile(5,true,8,summary) and (summary^.KiB=64) and summary^.RandomRead,'auto medium random'); + Assert(not suiteProfile(6,true,8,summary),'auto has exactly six cases'); + Assert(suiteProfile(11,false,1,summary) and (summary^.KiB=1024) and (summary^.Depth=32),'compare retains full matrix'); + Assert(not suiteProfile(12,false,1,summary),'compare has exactly twelve cases'); + kfree(void(summary)); + end else Assert(false,'suite test allocation'); + run := PBench(kalloc(sizeof(TBench))); device := PStorage_Device(kalloc(sizeof(TStorage_Device))); + if (run=nil) or (device=nil) then begin + if run<>nil then kfree(void(run)); if device<>nil then kfree(void(device)); + io.syslog.logln('STORBENCH FAIL','test allocation'); exit; + end; + memset(uint32(run),0,sizeof(TBench)); memset(uint32(device),0,sizeof(TStorage_Device)); + device^.dispatchRead := @testDispatch; device^.sectorSize := 512; device^.maxSectorCount := $FFFFFFFF; + Assert(configure(run,device,4096,1024,32,0,true,false) and (run^.Span=8388608),'4 GiB transfer does not wrap'); + Assert(not configure(run,device,1,1024,33,0,true,false),'depth memory cap'); + Assert(not configure(run,device,1,0,1,0,true,false),'zero request rejected'); + Assert(not configure(run,device,1,4,1,$FFFFFFFF,true,false),'start beyond device rejected'); + Assert(configure(run,device,1,4,1,$FFFFFFFE,true,false) and (run^.Span=1),'range clamp near uint32 end'); + slot := @run^.Slots[0]; planRead(run,slot); + Assert((slot^.LBA=$FFFFFFFE) and (slot^.Sectors=1) and (run^.Remaining=0),'short final request'); + device^.maxSectorCount := 4096; + Assert(configure(run,device,1,4,4,0,false,true) and (run^.StartLBA=2048),'default starting LBA'); + for i := 1 to 256 do begin + planRead(run,slot); + Assert((slot^.LBA>=2048) and (slot^.LBA<=4096-slot^.Sectors),'random request stays in range'); + end; + Assert(run^.Remaining=0,'random workload has exact sector count'); + device^.sectorSize := 2048; + Assert(not configure(run,device,1,1,1,0,true,false),'request sector alignment'); + run^.Completed := 20; run^.Histogram[0] := 18; run^.Histogram[2] := 1; run^.Histogram[10] := 1; + Assert(p95Ticks(run)=3,'95th percentile excludes slowest five percent'); + { Synthetic driver only: callbacks arrive inline in reverse groups of four. } + mock := PBenchMock(kalloc(sizeof(TBenchMock))); buffer := kalloc(4*65536); + device^.requestQueue := CFIFO_New(sizeof(pointer),8); + if (mock<>nil) and (buffer<>nil) and (device^.requestQueue<>nil) then begin + memset(uint32(mock),0,sizeof(TBenchMock)); mock^.Batch := 4; mock^.FailLBA := $FFFFFFFF; + device^.controllerId0 := uint32(mock); device^.sectorSize := 512; device^.maxActive := 4; + memset(uint32(run),0,sizeof(TBench)); run^.Buffer := buffer; + Assert(configure(run,device,1,64,4,0,true,false),'configure queued mock'); + execute(run); + Assert((run^.Accepted=16) and (run^.Completed=16) and (run^.Bytes=1048576),'queued reads complete exact byte count'); + Assert((run^.Peak=4) and (mock^.Count=0) and (run^.Active=0),'reverse groups drain at configured depth'); + Assert(not mock^.InvalidRange,'benchmark issues only in-range reads'); + memset(uint32(run),0,sizeof(TBench)); run^.Buffer := buffer; mock^.FailLBA := 128; + Assert(configure(run,device,1,64,4,0,true,false),'configure failing mock'); + execute(run); + Assert((run^.Error=eIOError) and (run^.Accepted=4) and (run^.Completed=4),'failure stops new submissions and drains accepted requests'); + Assert((run^.Bytes=3*65536) and (run^.Active=0),'failed reads excluded from throughput'); + memset(uint32(run),0,sizeof(TBench)); run^.Buffer := buffer; mock^.FailLBA := $FFFFFFFF; mock^.Batch := 1; + Assert(configure(run,device,1,64,1,0,true,false),'configure immediate mock'); + execute(run); + Assert((run^.Peak=1) and (run^.Bytes=1048576),'inline completions preserve peak and byte accounting'); + memset(uint32(run),0,sizeof(TBench)); run^.Buffer := buffer; + Assert(configure(run,device,1,64,4,0,true,false),'configure removed device mock'); + device^.removed := true; execute(run); + Assert((run^.Error=eDeviceRemoved) and (run^.Accepted=0) and (run^.Active=0),'submission rejection has no completion ownership'); + end else Assert(false,'mock allocation'); + if mock<>nil then kfree(void(mock)); if buffer<>nil then kfree(buffer); + if device^.requestQueue<>nil then CFIFO_Free(device^.requestQueue); + kfree(void(device)); kfree(void(run)); + io.syslog.log('STORBENCH','Passed: '); io.syslog.writeint(passed); + io.syslog.writestring(' Failed: '); io.syslog.writeintln(failed); +end; +procedure init; +begin registerCommand('STORBENCH',@command,'Read-only disk benchmark. STORBENCH help for sizes, queue depths and comparison mode.'); end; +initialization + boot.mgr.registerBoot('driver.storage.bench',@init,'Storage Benchmark',BOOT_MGR_BARRIER_LATE); +end. diff --git a/src/driver/storage/driver.storage.test.pas b/src/driver/storage/driver.storage.test.pas index 8400c885..b99a8a7f 100644 --- a/src/driver/storage/driver.storage.test.pas +++ b/src/driver/storage/driver.storage.test.pas @@ -50,27 +50,6 @@ uses driver.storage.vfs, driver.storage.vol.mgr; -const - STORBENCH_DEFAULT_MB = 32; - STORBENCH_BUFFER_BYTES = 1024 * 1024; - STORBENCH_TICK_HZ = 1024; - -function isNumericStr(s : pchar) : boolean; -var - i : uint32; -begin - isNumericStr := false; - if (s = nil) or (s[0] = char(0)) then - exit; - i := 0; - while s[i] <> char(0) do begin - if (s[i] < '0') or (s[i] > '9') then - exit; - i := i + 1; - end; - isNumericStr := true; -end; - { ============================================================ run_tests — shared test body called by both UnitTest (boot) and the STORTEST shell command (runtime). @@ -496,156 +475,6 @@ begin kfree(void(fStr)); end; -procedure cmd_storbench(params : PParamList; stdin_buf, stdout_buf, stderr_buf : POutBuf); -var - disk : PStorage_Device; - devIdx : uint32; - mbToRead : uint32; - totalBytes : uint32; - buf : puint32; - chunkSectors: uint32; - sectorsLeft : uint32; - sectorsNow : uint32; - startLBA : uint32; - lba : uint32; - t0, t1 : uint32; - tLast : uint32; - elapsed : uint32; - bytesInSec : uint32; - doneBytes : uint32; - kbps : uint32; - err : TError; - - procedure writeThroughput(kbpsValue : uint32); - begin - if kbpsValue >= 1024 then begin - io.stdio.bufWriteInt(stdout_buf, kbpsValue div 1024); - io.stdio.bufWriteStr(stdout_buf, '.'); - io.stdio.bufWriteInt(stdout_buf, ((kbpsValue mod 1024) * 10) div 1024); - io.stdio.bufWriteStr(stdout_buf, ' MB/s'); - end else begin - io.stdio.bufWriteInt(stdout_buf, kbpsValue); - io.stdio.bufWriteStr(stdout_buf, ' KB/s'); - end; - end; -begin - devIdx := 0; - mbToRead := STORBENCH_DEFAULT_MB; - - if paramCount(params) >= 1 then begin - if not isNumericStr(getParam(0, params)) then begin - io.stdio.bufWriteStrLn(stderr_buf, 'Usage: STORBENCH [device_index] [mb]'); - exit; - end; - devIdx := stringToInt(getParam(0, params)); - end; - - if paramCount(params) >= 2 then begin - if not isNumericStr(getParam(1, params)) then begin - io.stdio.bufWriteStrLn(stderr_buf, 'Usage: STORBENCH [device_index] [mb]'); - exit; - end; - mbToRead := stringToInt(getParam(1, params)); - if mbToRead = 0 then - mbToRead := STORBENCH_DEFAULT_MB; - end; - - disk := driver.storage.mgr.get_device(devIdx); - if disk = nil then begin - io.stdio.bufWriteStr(stderr_buf, 'No storage device at index '); - io.stdio.bufWriteIntLn(stderr_buf, devIdx); - exit; - end; - - if disk^.sectorSize = 0 then begin - io.stdio.bufWriteStrLn(stderr_buf, 'Device has invalid sector size.'); - exit; - end; - - totalBytes := mbToRead * 1024 * 1024; - chunkSectors := STORBENCH_BUFFER_BYTES div disk^.sectorSize; - if chunkSectors = 0 then - chunkSectors := 1; - - buf := puint32(kalloc(chunkSectors * disk^.sectorSize)); - if buf = nil then begin - io.stdio.bufWriteStrLn(stderr_buf, 'Out of memory allocating benchmark buffer.'); - exit; - end; - - sectorsLeft := totalBytes div disk^.sectorSize; - if sectorsLeft = 0 then - sectorsLeft := 1; - if sectorsLeft > disk^.maxSectorCount then - sectorsLeft := disk^.maxSectorCount; - - if disk^.maxSectorCount > (sectorsLeft + 2048) then - startLBA := 2048 - else - startLBA := 0; - lba := startLBA; - - io.stdio.bufWriteStr(stdout_buf, 'Raw read benchmark: disk '); - io.stdio.bufWriteInt(stdout_buf, devIdx); - io.stdio.bufWriteStr(stdout_buf, ', sectorSize='); - io.stdio.bufWriteInt(stdout_buf, disk^.sectorSize); - io.stdio.bufWriteStr(stdout_buf, ', maxActive='); - io.stdio.bufWriteInt(stdout_buf, disk^.maxActive); - io.stdio.bufWriteNewLine(stdout_buf); - - t0 := arch.x86.bda.Counters.c32; - tLast := t0; - bytesInSec := 0; - doneBytes := 0; - - while sectorsLeft > 0 do begin - sectorsNow := chunkSectors; - if sectorsNow > sectorsLeft then - sectorsNow := sectorsLeft; - - err := driver.storage.mgr.storage_read(disk, lba, sectorsNow, buf); - if err <> eNone then begin - io.stdio.bufWriteStr(stderr_buf, 'Raw read failed at LBA '); - io.stdio.bufWriteInt(stderr_buf, lba); - io.stdio.bufWriteStr(stderr_buf, ' error='); - io.stdio.bufWriteIntLn(stderr_buf, ord(err)); - kfree(void(buf)); - exit; - end; - - lba := lba + sectorsNow; - sectorsLeft := sectorsLeft - sectorsNow; - doneBytes := doneBytes + (sectorsNow * disk^.sectorSize); - bytesInSec := bytesInSec + (sectorsNow * disk^.sectorSize); - - t1 := arch.x86.bda.Counters.c32; - if (t1 - tLast) >= STORBENCH_TICK_HZ then begin - kbps := (bytesInSec div 1024) * STORBENCH_TICK_HZ div (t1 - tLast); - io.stdio.bufWriteStr(stdout_buf, ' RAW '); - io.stdio.bufWriteInt(stdout_buf, doneBytes div 1024); - io.stdio.bufWriteStr(stdout_buf, ' KB '); - writeThroughput(kbps); - io.stdio.bufWriteNewLine(stdout_buf); - tLast := t1; - bytesInSec := 0; - end; - end; - - t1 := arch.x86.bda.Counters.c32; - elapsed := t1 - t0; - io.stdio.bufWriteStr(stdout_buf, 'Raw read complete: '); - if elapsed > 0 then begin - kbps := ((doneBytes div 1024) * STORBENCH_TICK_HZ) div elapsed; - writeThroughput(kbps); - io.stdio.bufWriteStr(stdout_buf, ' ('); - io.stdio.bufWriteInt(stdout_buf, (elapsed * 1000) div STORBENCH_TICK_HZ); - io.stdio.bufWriteStrLn(stdout_buf, ' ms)'); - end else - io.stdio.bufWriteStrLn(stdout_buf, '< 1 tick'); - - kfree(void(buf)); -end; - { ============================================================ init — register the storage terminal commands ============================================================ } @@ -656,8 +485,6 @@ begin 'Run storage subsystem unit tests.'); io.stdio.registerCommand('DISKTEST', @cmd_disktest, 'DESTRUCTIVE: wipe disk 0, format FAT32 4MB, write+read file.'); - io.stdio.registerCommand('STORBENCH', @cmd_storbench, - 'Read-only raw storage benchmark: STORBENCH [device_index] [mb].'); debug.tracer.pop_trace; end; diff --git a/src/io/io.stdio.pas b/src/io/io.stdio.pas index fb287cd5..e90cd5b5 100644 --- a/src/io/io.stdio.pas +++ b/src/io/io.stdio.pas @@ -486,6 +486,10 @@ var begin search := params; for i := 0 to index do begin + if search = nil then begin + getParam := nil; + exit; + end; if search^.next = nil then begin getParam := nil; exit; diff --git a/src/proc/proc.mgr.pas b/src/proc/proc.mgr.pas index 83dfb486..68d78563 100644 --- a/src/proc/proc.mgr.pas +++ b/src/proc/proc.mgr.pas @@ -107,10 +107,10 @@ procedure process_trampoline; forward; { ---- Process creation ---- } -function create(name : pchar; +function createInternal(name : pchar; entry : TProcessEntryPoint; localData : void; - priority : uint8) : PProcessContext; + priority : uint8; runnable : boolean) : PProcessContext; var ctx : PProcessContext; slotPtr : ^uint32; @@ -119,7 +119,7 @@ var prio : uint8; begin push_trace('proc.mgr.create'); - create := nil; + createInternal := nil; if priority = 0 then prio := 1 else prio := priority; @@ -140,7 +140,7 @@ begin ctx^.ParentID := CurrentProcess^.ProcessID; { State } - ctx^.State := psCreated; + ctx^.State := psSuspended; ctx^.ExitCode := 0; { Entry point } @@ -269,14 +269,22 @@ begin slotPtr^ := uint32(ctx); { Mark as ready for scheduling } - ctx^.State := psReady; + if runnable then ctx^.State := psReady; asm popf end; io.syslog.log('PROCMGR', 'Created process: '); io.syslog.writestringln(name); pop_trace; - create := ctx; + createInternal := ctx; +end; + +{ Public creation retains its ready-on-return behavior. Commands need their + arguments and streams installed before the scheduler may enter them. } +function create(name : pchar; entry : TProcessEntryPoint; + localData : void; priority : uint8) : PProcessContext; +begin + create := createInternal(name, entry, localData, priority, true); end; { ---- Process trampoline ---- } @@ -543,12 +551,17 @@ function runCommand(name : pchar; method : TCommandMethod; params : PParamList; var ctx : PProcessContext; begin - ctx := create(name, @legacy_entry, void(method), 1); + ctx := createInternal(name, @legacy_entry, void(method), 1, false); if ctx <> nil then begin ctx^.MsgData := void(params); ctx^.StdIn := stdin_buf; ctx^.StdOut := stdout_buf; ctx^.StdErr := stderr_buf; + { Publish only after all fields read by legacy_entry are initialized. + psCreated is also runnable, so creation must use psSuspended. } + asm pushf; cli end; + ctx^.State := psReady; + asm popf end; end; runCommand := ctx; end; diff --git a/src/test/test.pas b/src/test/test.pas index b9ddd739..ea33ad07 100644 --- a/src/test/test.pas +++ b/src/test/test.pas @@ -33,6 +33,7 @@ uses core.ds.prio, driver.storage.vfs, driver.storage.test, + driver.storage.bench, driver.storage.metatest, driver.storage.perftest, driver.storage.ctl.ahci, @@ -71,6 +72,7 @@ begin core.ds.prio.UnitTest; driver.storage.vfs.UnitTest; driver.storage.test.UnitTest; + driver.storage.bench.UnitTest; driver.storage.metatest.UnitTest; driver.storage.perftest.UnitTest; driver.storage.ctl.ahci.UnitTest;