From 8e0ab3da8b32f4054a5ca4978db389b88b108b95 Mon Sep 17 00:00:00 2001 From: Aaron Date: Sat, 31 Oct 2026 17:32:59 +0000 Subject: [PATCH] Improve storage performance with read pipelining, run lookup, request merging, and counters --- doc/src/driver/storage/driver.storage.mgr.md | 34 ++++ src/app/app.diskcmd.pas | 11 + .../ctl/ahci/driver.storage.ctl.ahci.pas | 1 + .../storage/driver.storage.iorequest.pas | 10 +- src/driver/storage/driver.storage.mgr.pas | 111 +++++++--- .../storage/driver.storage.perftest.pas | 189 ++++++++++++++++++ src/driver/storage/driver.storage.types.pas | 13 ++ .../fs/fat32/driver.storage.fs.fat32.core.pas | 138 +++++++------ .../driver.storage.fs.fat32.transfer.pas | 139 +++++++++++-- .../fat32/driver.storage.fs.fat32.types.pas | 12 ++ src/test/test.pas | 2 + 11 files changed, 546 insertions(+), 114 deletions(-) create mode 100644 src/driver/storage/driver.storage.perftest.pas diff --git a/doc/src/driver/storage/driver.storage.mgr.md b/doc/src/driver/storage/driver.storage.mgr.md index 7b3c84cf..f92a72d6 100644 --- a/doc/src/driver/storage/driver.storage.mgr.md +++ b/doc/src/driver/storage/driver.storage.mgr.md @@ -219,3 +219,37 @@ Drop-in procedure replacements for callers using the old void-returning API. For - `dispatch_next` is an internal procedure called with interrupts disabled. It dequeues the head of the CFIFO, allocates a heap copy via `ioreq_copy` (for ISR safety), and invokes the appropriate `dispatchRead` or `dispatchWrite` on the device. If neither dispatch pointer is set, the request is failed with `eDeviceNotFound`. - The blocking path relies on pointer-dereferenced spin (`puint32(@wait.Done)^`) to prevent FPC from caching the `Done` flag in a register across the `hlt` instruction. - Devices without `dispatchRead`/`dispatchWrite` set (i.e., drivers not yet migrated to Phase 2) will have `submit_io` return `eDeviceNotFound`. Legacy drivers must be migrated before they can use the new I/O path. + + +## Bounded performance path + +The manager merges up to 32 consecutive queued requests when operation, device, +LBA range, and virtual buffer range are contiguous. It never searches past the +queue head or delays a lone request to create a merge. Drivers opt in through +`mergeMaxSectors`; SATA AHCI advertises 65,536 sectors, while other drivers default +to zero (disabled). A merged command delivers one completion to each original +request, in FIFO order. A command error fails every member conservatively. + +`storage_get_stats` takes an interrupt-protected snapshot of the device counters. +`DISK info ` displays accepted/completed/failed logical requests, hardware +commands, merged requests, queue-full rejections, queue and active high-water +marks, successful read/write sectors, request-size buckets, queue/service ticks, +and FAT metadata cache hits/misses. Queue ticks are accumulated per logical +request; service ticks are accumulated per hardware command. Tick values use +`Counters.c32`, not milliseconds. Counters are unsigned 32-bit values, start at +device registration, and wrap; use snapshot differences over bounded benchmark +intervals. Size buckets are <=8, 9..256, and >256 sectors. + +FAT32 blocking and async reads share a pooled transfer engine with batches of up +to four outstanding reads, capped by device queue depth. Destination slices are +disjoint. On any error, the engine drains every accepted request before releasing +its context and reports only the contiguous successful byte prefix. Unaligned +edges and odd DMA addresses use the context's private scratch sector. Writes +remain sequential. Run maps store cumulative file-cluster offsets; an open-file +cursor handles sequential locality and binary search handles random seeks. + +`STORPERF` runs memory-backed regression checks (also included in boot unit tests) +for reverse completion order, failures, unaligned reads, run lookup, queue +rejection, merging limits, and callback accounting. It does not access real disks +or measure hardware throughput. Compare counters alongside `STORBENCH` and file +benchmarks on contiguous and fragmented files for actual performance validation. diff --git a/src/app/app.diskcmd.pas b/src/app/app.diskcmd.pas index d21dc43a..ec73b5b4 100644 --- a/src/app/app.diskcmd.pas +++ b/src/app/app.diskcmd.pas @@ -150,6 +150,7 @@ var device : PStorage_Device; volCount : uint32; freeSec : uint32; + stats : TStorageStats; begin if paramCount(params) < 2 then begin println('Usage: DISK info '); @@ -191,6 +192,16 @@ begin else volCount := 0; print('Partitions: '); printint(volCount); println(''); + storage_get_stats(device,stats); + print('I/O submitted/completed/failed: '); printint(stats.Submitted); print(' / '); printint(stats.Completed); print(' / '); printint(stats.Failed); println(''); + print('Commands / merged requests: '); printint(stats.Commands); print(' / '); printint(stats.Merged); println(''); + print('Queue peak / active peak / queue full: '); printint(stats.QueuePeak); print(' / '); printint(stats.ActivePeak); print(' / '); printint(stats.QueueFull); println(''); + print('Read / written sectors: '); printint(stats.ReadSectors); print(' / '); printint(stats.WriteSectors); println(''); + print('Requests <=8 / <=256 / >256 sectors: '); printint(stats.SmallRequests); print(' / '); printint(stats.MediumRequests); print(' / '); printint(stats.LargeRequests); println(''); + print('Queue ticks total / max: '); printint(stats.QueueTicks); print(' / '); printint(stats.MaxQueueTicks); println(''); + print('Service ticks total / max: '); printint(stats.ServiceTicks); print(' / '); printint(stats.MaxServiceTicks); println(''); + print('FAT cache hits / misses: '); printint(stats.FatCacheHits); print(' / '); printint(stats.FatCacheMisses); println(''); + end; { ---------- DISK WIPE ---------- } diff --git a/src/driver/storage/ctl/ahci/driver.storage.ctl.ahci.pas b/src/driver/storage/ctl/ahci/driver.storage.ctl.ahci.pas index 0e6fe06c..fd1fff29 100644 --- a/src/driver/storage/ctl/ahci/driver.storage.ctl.ahci.pas +++ b/src/driver/storage/ctl/ahci/driver.storage.ctl.ahci.pas @@ -397,6 +397,7 @@ begin storageDev^.controllerId0 := uint32(device); { store PAHCI_Device for callback wrappers } storageDev^.sectorSize := storageDev_sectorSize; storageDev^.maxSectorCount := sec_count; + if not isATAPI then storageDev^.mergeMaxSectors := 65536; storageDev^.start := 0; storageDev^.volumes := DL_New(sizeof(Pointer)); diff --git a/src/driver/storage/driver.storage.iorequest.pas b/src/driver/storage/driver.storage.iorequest.pas index daf680f1..293b2e11 100644 --- a/src/driver/storage/driver.storage.iorequest.pas +++ b/src/driver/storage/driver.storage.iorequest.pas @@ -39,8 +39,8 @@ function ioreq_alloc(reqType : TIORequestType; procedure ioreq_free(req : PIORequest); { Copy an existing TIORequest into a new pooled allocation (deep copy of the - record, NOT the buffer it points to). Used by dispatch_next to create an - ISR-safe copy from the CFIFO value-element. } + record, NOT the buffer it points to). Used by submit_io before enqueueing + the owned pointer. Manager-only merge/timing fields are initialised here. } function ioreq_copy(src : PIORequest) : PIORequest; implementation @@ -132,6 +132,9 @@ begin ioreq_alloc := nil; exit; end; + req^.MergeNext := nil; + req^.OriginalSectors := sectors; + req^.QueuedTick := 0; req^.DispatchTick := 0; req^.RequestType := reqType; req^.State := iosPending; req^.Device := device; @@ -161,6 +164,9 @@ begin ioreq_copy := nil; exit; end; + dst^.MergeNext := nil; + dst^.OriginalSectors := src^.SectorCount; + dst^.QueuedTick := 0; dst^.DispatchTick := 0; dst^.RequestType := src^.RequestType; dst^.State := src^.State; dst^.Device := src^.Device; diff --git a/src/driver/storage/driver.storage.mgr.pas b/src/driver/storage/driver.storage.mgr.pas index 4e1dcb4a..588e7d8d 100644 --- a/src/driver/storage/driver.storage.mgr.pas +++ b/src/driver/storage/driver.storage.mgr.pas @@ -35,6 +35,7 @@ unit driver.storage.mgr; interface uses + arch.x86.bda, boot.mgr, core.ds.cfifo, core.ds.types, @@ -59,6 +60,7 @@ var hdCount : uint32; { non-ATAPI devices registered so far } atapiCount : uint32; { ATAPI devices registered so far } +procedure storage_get_stats(device : PStorage_Device; var stats : TStorageStats); procedure init(); procedure set_boot_drive_byte(driveByte : uint8); function get_boot_drive_byte() : uint8; @@ -212,6 +214,14 @@ begin kfree(void(ctx)); end; +procedure storage_get_stats(device : PStorage_Device; var stats : TStorageStats); +begin + asm pushf; cli end; + if device <> nil then stats := device^.Stats + else memset(uint32(@stats),0,sizeof(stats)); + asm popf end; +end; + procedure init(); begin push_trace('driver.storage.mgr.init'); @@ -245,6 +255,7 @@ begin { Initialise new Phase 2 fields } device^.removed := false; device^.requestQueue := CFIFO_New(SizeOf(PIORequest), 128); + memset(uint32(@device^.Stats),0,sizeof(TStorageStats)); device^.activeCount := 0; device^.dispatching := false; if device^.maxActive = 0 then device^.maxActive := 1; @@ -415,7 +426,9 @@ end; Called with interrupts disabled (CLI context) or from ISR context. } procedure dispatch_next(device : PStorage_Device); var - heapReq : PIORequest; + heapReq, nextReq, tail : PIORequest; + peek : pointer; + merged, bytes, ticks : uint32; begin if device^.dispatching then exit; device^.dispatching := true; @@ -425,8 +438,41 @@ begin if heapReq = nil then break; + { Merge only consecutive FIFO entries with consecutive DMA buffers. + Bound both command size and completion fan-out; never reorder writes. } + tail := heapReq; merged := 1; + while (merged < 32) and (device^.mergeMaxSectors > 0) and + (device^.sectorSize > 0) do begin + peek := CFIFO_Peek(device^.requestQueue); + if peek = nil then break; + nextReq := PIORequest(puint32(peek)^); + if nextReq = nil then break; + if (nextReq^.Device <> device) or (nextReq^.RequestType <> heapReq^.RequestType) then break; + if heapReq^.SectorCount > device^.mergeMaxSectors then break; + if nextReq^.SectorCount > device^.mergeMaxSectors-heapReq^.SectorCount then break; + if heapReq^.SectorCount > ($FFFFFFFF div device^.sectorSize) then break; + bytes := heapReq^.SectorCount*device^.sectorSize; + if (heapReq^.LBA > $FFFFFFFF-heapReq^.SectorCount) or + (uint32(heapReq^.Buffer) > $FFFFFFFF-bytes) then break; + if (nextReq^.LBA <> heapReq^.LBA+heapReq^.SectorCount) or + (uint32(nextReq^.Buffer) <> uint32(heapReq^.Buffer)+bytes) then break; + if not CFIFO_Dequeue(device^.requestQueue,@nextReq) then break; + tail^.MergeNext := nextReq; tail := nextReq; + Inc(heapReq^.SectorCount,nextReq^.SectorCount); Inc(merged); + Inc(device^.Stats.Merged); + end; + ticks := puint32(@Counters.c32)^; + tail := heapReq; + while tail <> nil do begin + tail^.DispatchTick := ticks; tail^.State := iosDispatched; + bytes := ticks-tail^.QueuedTick; + Inc(device^.Stats.QueueTicks,bytes); + if bytes > device^.Stats.MaxQueueTicks then device^.Stats.MaxQueueTicks := bytes; + tail := tail^.MergeNext; + end; + Inc(device^.Stats.Commands); device^.activeCount := device^.activeCount + 1; - heapReq^.State := iosDispatched; + if device^.activeCount > device^.Stats.ActivePeak then device^.Stats.ActivePeak := device^.activeCount; case heapReq^.RequestType of ioRead: begin @@ -484,12 +530,20 @@ begin Reject once the fixed queue is full instead of growing with CLI held. } asm pushfd; cli; pop flags end; if CFIFO_Size(device^.requestQueue) >= CFIFO_Capacity(device^.requestQueue) then begin + Inc(device^.Stats.QueueFull); if (flags and $200) <> 0 then asm sti end; ioreq_free(heapReq); submit_io := eQueueFull; exit; end; + heapReq^.QueuedTick := puint32(@Counters.c32)^; + Inc(device^.Stats.Submitted); + if request^.SectorCount <= 8 then Inc(device^.Stats.SmallRequests) + else if request^.SectorCount <= 256 then Inc(device^.Stats.MediumRequests) + else Inc(device^.Stats.LargeRequests); CFIFO_Enqueue(device^.requestQueue, @heapReq); + if CFIFO_Size(device^.requestQueue) > device^.Stats.QueuePeak then + device^.Stats.QueuePeak := CFIFO_Size(device^.requestQueue); dispatch_next(device); if (flags and $200) <> 0 then asm sti end; @@ -545,36 +599,39 @@ begin end; procedure complete_io(request : PIORequest; success : boolean; error : TError); -var - device : PStorage_Device; +var device : PStorage_Device; next : PIORequest; ticks, flags : uint32; + wasDispatching : boolean; begin if request = nil then exit; - device := request^.Device; - - if success then begin - request^.State := iosComplete; - request^.Error := eNone; - end else begin - request^.State := iosError; - request^.Error := error; + asm pushfd; cli; pop flags end; + wasDispatching := false; + if device <> nil then begin + wasDispatching := device^.dispatching; + device^.dispatching := true; { callbacks cannot dispatch inside merged fan-out } + if device^.activeCount > 0 then Dec(device^.activeCount); + ticks := puint32(@Counters.c32)^-request^.DispatchTick; + Inc(device^.Stats.ServiceTicks,ticks); + if ticks > device^.Stats.MaxServiceTicks then device^.Stats.MaxServiceTicks := ticks; end; - - { Decrement active count on the device } - if device <> nil then - if device^.activeCount > 0 then - device^.activeCount := device^.activeCount - 1; - - { Invoke completion callback (sync path uses sync_io_callback, async path uses caller's callback) } - if request^.Callback <> nil then - request^.Callback(request^.Error, request^.CallbackData); - - { Free the heap copy of the request } - ioreq_free(request); - - { Dispatch next queued request for this device } - if device <> nil then + while request <> nil do begin + next := request^.MergeNext; + if success then begin request^.State := iosComplete; request^.Error := eNone; end + else begin request^.State := iosError; request^.Error := error; end; + if device <> nil then begin + Inc(device^.Stats.Completed); + if not success then Inc(device^.Stats.Failed) + else if request^.RequestType = ioRead then Inc(device^.Stats.ReadSectors,request^.OriginalSectors) + else Inc(device^.Stats.WriteSectors,request^.OriginalSectors); + end; + if request^.Callback <> nil then request^.Callback(request^.Error,request^.CallbackData); + ioreq_free(request); request := next; + end; + if device <> nil then begin + device^.dispatching := wasDispatching; dispatch_next(device); + end; + if (flags and $200) <> 0 then asm sti end; end; { ========================================================================== } diff --git a/src/driver/storage/driver.storage.perftest.pas b/src/driver/storage/driver.storage.perftest.pas new file mode 100644 index 00000000..5acf8901 --- /dev/null +++ b/src/driver/storage/driver.storage.perftest.pas @@ -0,0 +1,189 @@ +unit driver.storage.perftest; +interface +procedure UnitTest; +procedure init; +implementation +uses boot.mgr, io.stdio, io.syslog, memory.heap, core.util, core.ds.cfifo, + driver.storage.types, driver.storage.mgr, driver.storage.fs.fat32.types, + driver.storage.fs.fat32.core, driver.storage.fs.fat32.transfer; +type + PMock = ^TMock; + TMock = record + Data : puint8; + Pending : array[0..7] of PIORequest; + Count, Calls, FailLBA, Callbacks, Bytes : uint32; + Deferred : boolean; + Error : TError; + end; +procedure finish(request : PIORequest); +var m : PMock; bytes : uint32; +begin + m := PMock(request^.Device^.controllerId0); + if request^.LBA = m^.FailLBA then begin complete_io(request,false,eIOError); exit; end; + if (request^.LBA >= 128) or (request^.SectorCount > 128-request^.LBA) then begin + complete_io(request,false,eInvalidArgument); exit; + end; + if (uint32(request^.Buffer) and 1) <> 0 then begin complete_io(request,false,eInvalidArgument); exit; end; + bytes := request^.SectorCount*512; + if request^.RequestType = ioRead then memcpy(uint32(m^.Data)+request^.LBA*512,uint32(request^.Buffer),bytes) + else memcpy(uint32(request^.Buffer),uint32(m^.Data)+request^.LBA*512,bytes); + complete_io(request,true,eNone); +end; +procedure dispatch(device : PStorage_Device; request : PIORequest); +var m : PMock; +begin + m := PMock(device^.controllerId0); Inc(m^.Calls); + if m^.Deferred then begin + if m^.Count = 8 then begin complete_io(request,false,eQueueFull); exit; end; + m^.Pending[m^.Count] := request; Inc(m^.Count); + end else finish(request); +end; +procedure done(error : TError; data : pointer); +var m : PMock; +begin m := PMock(data); Inc(m^.Callbacks); m^.Error := error; end; +procedure drain(m : PMock); +var request : PIORequest; n : uint32; +begin + n := 0; + while (m^.Count > 0) and (n < 64) do begin + Inc(n); Dec(m^.Count); request := m^.Pending[m^.Count]; finish(request); + end; +end; +procedure UnitTest; +var device : PStorage_Device; volume : PStorage_Volume; ofi : PFATOpenFile; + m : PMock; vc : PFATVolumeCtx; br : PBootRecord; buffer : puint8; + i, j, idx, base, cluster, off, available, passed, failed, before : uint32; + ok : boolean; + queued : PIORequest; + procedure Assert(value : boolean; name : pchar); + begin + if value then Inc(passed) else begin Inc(failed); io.syslog.logln('STORPERF FAIL',name); end; + end; +begin + passed := 0; failed := 0; + device := PStorage_Device(kalloc(sizeof(TStorage_Device))); + volume := PStorage_Volume(kalloc(sizeof(TStorage_Volume))); + ofi := PFATOpenFile(kalloc(sizeof(TFATOpenFile))); + m := PMock(kalloc(sizeof(TMock))); buffer := puint8(kalloc(4096)); + if (device=nil) or (volume=nil) or (ofi=nil) or (m=nil) or (buffer=nil) then begin + if device<>nil then kfree(void(device)); if volume<>nil then kfree(void(volume)); + if ofi<>nil then kfree(void(ofi)); if m<>nil then kfree(void(m)); if buffer<>nil then kfree(void(buffer)); + io.syslog.logln('STORPERF FAIL','allocation'); exit; + end; + memset(uint32(device),0,sizeof(TStorage_Device)); memset(uint32(volume),0,sizeof(TStorage_Volume)); + memset(uint32(ofi),0,sizeof(TFATOpenFile)); memset(uint32(m),0,sizeof(TMock)); + m^.Data := puint8(kalloc(128*512)); ofi^.RunMap.Runs := PFATRun(kalloc(4*sizeof(TFATRun))); + if (m^.Data=nil) or (ofi^.RunMap.Runs=nil) then begin + if m^.Data<>nil then kfree(void(m^.Data)); if ofi^.RunMap.Runs<>nil then kfree(void(ofi^.RunMap.Runs)); + kfree(void(device)); kfree(void(volume)); kfree(void(ofi)); kfree(void(m)); kfree(void(buffer)); + io.syslog.logln('STORPERF FAIL','data allocation'); exit; + end; + for i := 0 to 127 do memset(uint32(m^.Data)+i*512,i,512); + br := PBootRecord(m^.Data); memset(uint32(br),0,sizeof(TBootRecord)); + br^.sectorSize := 512; br^.spc := 1; br^.rsvSectors := 1; br^.numFats := 1; + br^.FATSize := 1; br^.manySectors := 128; br^.rootCluster := 2; + device^.controllerId0 := uint32(m); device^.sectorSize := 512; device^.maxSectorCount := 128; + device^.writable := true; device^.maxActive := 4; + device^.dispatchRead := @dispatch; device^.dispatchWrite := @dispatch; + device^.requestQueue := CFIFO_New(sizeof(pointer),16); m^.FailLBA := $FFFFFFFF; + volume^.device := device; volume^.sectorCount := 128; volume^.sectorSize := 512; + vc := FAT32GetVolumeCtx(volume); + Assert(vc<>nil,'synthetic volume context'); + if vc<>nil then begin + ofi^.Volume := volume; ofi^.VolumeInfo := @vc^.Info; ofi^.ByteSize := 4096; + ofi^.AllocClusters := 8; ofi^.RunMap.Count := 4; ofi^.RunMap.Capacity := 4; + ofi^.RunMap.TotalClusters := 8; ofi^.Exists := true; ofi^.Registered := true; ofi^.DirLoc.Valid := true; + ofi^.ScratchSector := puint32(kalloc(512)); ofi^.ScratchSize := 512; + for i := 0 to 3 do begin + ofi^.RunMap.Runs[i].StartCluster := 10+i*10; + ofi^.RunMap.Runs[i].ClusterCount := 2; + ofi^.RunMap.Runs[i].FileClusterStart := i*2; + end; + Assert(FAT32FindRunForOffset(ofi,3584,false,0,0,idx,base,cluster,off,available) and + (idx=3) and (cluster=41) and (available=512),'random run seek'); + Assert(FAT32FindRunForOffset(ofi,512,false,0,0,idx,base,cluster,off,available) and + (idx=0) and (cluster=11),'backward persistent cursor seek'); + Assert(not FAT32FindRunForOffset(ofi,4096,false,0,0,idx,base,cluster,off,available),'run EOF'); + m^.Deferred := true; m^.Callbacks := 0; + FAT32ReadFileAtOffsetAsync(volume,nil,nil,0,puint32(buffer),4096,ofi,@m^.Bytes,@done,m); + Assert((m^.Count=4) and (m^.Callbacks=0),'four reads outstanding'); + Dec(m^.Count); finish(m^.Pending[m^.Count]); + Assert(m^.Callbacks=0,'reverse completion retains context'); drain(m); + Assert((m^.Callbacks=1) and (m^.Bytes=4096) and (m^.Error=eNone),'batch completes once'); + ok := true; + for i := 0 to 4095 do if buffer[i] <> 10+(i div 1024)*10+(i mod 1024) div 512 then ok := false; + Assert(ok,'fragmented DMA contents and cluster offsets'); + m^.Callbacks := 0; m^.FailLBA := 20; + FAT32ReadFileAtOffsetAsync(volume,nil,nil,0,puint32(buffer),4096,ofi,@m^.Bytes,@done,m); drain(m); + Assert((m^.Callbacks=1) and (m^.Bytes=1024) and (m^.Error=eIOError) and (device^.activeCount=0),'failure drains DMA and reports prefix'); + m^.FailLBA := $FFFFFFFF; m^.Callbacks := 0; + FAT32ReadFileAtOffsetAsync(volume,nil,nil,513,puint32(buffer),3073,ofi,@m^.Bytes,@done,m); drain(m); + ok := (m^.Callbacks=1) and (m^.Bytes=3073) and (m^.Error=eNone); + for i := 0 to 3072 do begin j := i+513; + if buffer[i] <> 10+(j div 1024)*10+(j mod 1024) div 512 then ok := false; + end; + Assert(ok,'unaligned head and tail around pipeline'); + m^.Deferred := false; m^.Callbacks := 0; + FAT32ReadFileAtOffsetAsync(volume,nil,nil,0,puint32(buffer),4096,ofi,@m^.Bytes,@done,m); + Assert((m^.Callbacks=1) and (m^.Bytes=4096) and (m^.Error=eNone),'inline batch completion is iterative'); + m^.FailLBA := 10; m^.Callbacks := 0; + FAT32ReadFileAtOffsetAsync(volume,nil,nil,0,puint32(buffer),4096,ofi,@m^.Bytes,@done,m); + Assert((m^.Callbacks=1) and (m^.Bytes=0) and (m^.Error=eIOError),'inline first-read failure completes once'); + m^.FailLBA := $FFFFFFFF; + device^.maxActive := 0; m^.Callbacks := 0; + for i := 0 to 15 do Assert(storage_read_async(device,50,1,puint32(buffer),nil,nil)=eNone,'fill fixed queue'); + FAT32ReadFileAtOffsetAsync(volume,nil,nil,0,puint32(buffer),4096,ofi,@m^.Bytes,@done,m); + Assert((m^.Callbacks=1) and (m^.Bytes=0) and (m^.Error=eQueueFull),'rejected pipeline submission releases context'); + device^.maxActive := 4; + { Simulate dispatch of the first queued request, then let completion drain. } + if CFIFO_Dequeue(device^.requestQueue,@queued) then begin + Inc(device^.activeCount); finish(queued); + end; + Assert(CFIFO_Size(device^.requestQueue)=0,'full queue drained'); + Assert(FAT32ReadFileAtOffset(volume,nil,nil,512,puint32(buffer),512,ofi)=512,'blocking read uses pipeline'); + Assert(buffer[0]=11,'offset inside contiguous run is not counted twice'); + m^.FailLBA := 10; + Assert(FAT32WriteFileAtOffset(volume,nil,nil,0,puint32(buffer),512,ofi)=0,'failed write reports zero bytes'); + m^.FailLBA := $FFFFFFFF; + if ofi^.ScratchSector<>nil then kfree(void(ofi^.ScratchSector)); + FAT32ReleaseVolumeCtx(volume); + end; + { Force a backlog without issuing hardware I/O, then drain through a mock. } + device^.maxActive := 0; device^.mergeMaxSectors := 8; m^.Deferred := false; + m^.Callbacks := 0; before := m^.Calls; + for i := 0 to 3 do Assert(storage_read_async(device,10+i,1,puint32(uint32(buffer)+i*512),@done,m)=eNone,'merge enqueue'); + device^.maxActive := 1; + Assert(storage_read_async(device,50,1,puint32(buffer),@done,m)=eNone,'drain trigger'); + Assert((m^.Calls-before=2) and (m^.Callbacks=5),'four requests merge with five callbacks including trigger'); + Assert(device^.Stats.Merged=3,'merge counter'); + device^.maxActive := 0; m^.Callbacks := 0; m^.FailLBA := 10; + for i := 0 to 1 do Assert(storage_write_async(device,10+i,1,puint32(uint32(buffer)+i*512),@done,m)=eNone,'merged write enqueue'); + device^.maxActive := 1; before := device^.Stats.Failed; + Assert(storage_read_async(device,50,1,puint32(buffer),@done,m)=eNone,'failed merge drain trigger'); + Assert((device^.Stats.Failed-before=2) and (m^.Callbacks=3),'merged failure fans out once'); + m^.FailLBA := $FFFFFFFF; device^.maxActive := 0; device^.mergeMaxSectors := 2; + m^.Callbacks := 0; before := m^.Calls; + for i := 0 to 3 do Assert(storage_read_async(device,10+i,1,puint32(uint32(buffer)+i*512),@done,m)=eNone,'limited merge enqueue'); + device^.maxActive := 1; + Assert(storage_read_async(device,50,1,puint32(buffer),@done,m)=eNone,'limited merge drain trigger'); + Assert((m^.Calls-before=3) and (m^.Callbacks=5),'driver command limit splits merges'); + device^.maxActive := 0; device^.mergeMaxSectors := 8; before := m^.Calls; + Assert(storage_read_async(device,10,1,puint32(buffer),nil,nil)=eNone,'ordering read'); + Assert(storage_write_async(device,11,1,puint32(uint32(buffer)+512),nil,nil)=eNone,'ordering write'); + Assert(storage_write_async(device,12,1,puint32(buffer),nil,nil)=eNone,'noncontiguous buffer'); + device^.maxActive := 1; + Assert(storage_read_async(device,50,1,puint32(buffer),nil,nil)=eNone,'ordering drain trigger'); + Assert(m^.Calls-before=4,'operation and buffer boundaries prevent merging'); + Assert(device^.Stats.Completed=device^.Stats.Submitted,'all accepted requests accounted'); + CFIFO_Free(device^.requestQueue); kfree(void(ofi^.RunMap.Runs)); kfree(void(m^.Data)); + kfree(void(device)); kfree(void(volume)); kfree(void(ofi)); kfree(void(m)); kfree(void(buffer)); + io.syslog.log('STORPERF','Passed: '); io.syslog.writeint(passed); + io.syslog.writestring(' Failed: '); io.syslog.writeintln(failed); +end; +procedure command(params : PParamList; stdin_buf, stdout_buf, stderr_buf : POutBuf); +begin UnitTest; io.stdio.bufWriteStrLn(stdout_buf,'Storage performance regression results written to serial log.'); end; +procedure init; +begin io.stdio.registerCommand('STORPERF',@command,'Run memory-backed storage performance regression tests'); end; +initialization + boot.mgr.registerBoot('driver.storage.perftest',@init,'Storage Performance Tests',BOOT_MGR_BARRIER_MID); +end. diff --git a/src/driver/storage/driver.storage.types.pas b/src/driver/storage/driver.storage.types.pas index 9bc82597..936ea5d3 100644 --- a/src/driver/storage/driver.storage.types.pas +++ b/src/driver/storage/driver.storage.types.pas @@ -252,11 +252,22 @@ type UserData : pointer; { points back to caller's stack-local request (blocking path) } Callback : TIOCallback; { async completion callback (nil for blocking path) } CallbackData : pointer; { opaque data passed to Callback } + { Manager-owned fields; callers need not initialise these. } + MergeNext : PIORequest; + OriginalSectors, QueuedTick, DispatchTick : uint32; end; byteArray8 = array[0..7] of char; PByteArray8 = ^byteArray8; + TStorageStats = record + Submitted, Completed, Failed, Commands, Merged, QueueFull : uint32; + ReadSectors, WriteSectors, QueuePeak, ActivePeak : uint32; + QueueTicks, ServiceTicks, MaxQueueTicks, MaxServiceTicks : uint32; + SmallRequests, MediumRequests, LargeRequests : uint32; + FatCacheHits, FatCacheMisses : uint32; + end; + { Generic storage device } TStorage_Device = record id : uint8; @@ -272,6 +283,7 @@ type requestQueue : PCFIFOQueue; { per-device I/O request CFIFO } dispatching : boolean; { prevents recursive synchronous-error queue draining } activeCount : uint32; { number of commands currently dispatched to HW } + mergeMaxSectors : uint32; { zero disables merging; driver sets safe command limit } maxActive : uint32; { max concurrent commands (NCQ depth) } maxSectorCount : uint32; @@ -281,6 +293,7 @@ type mbrWriteBusy : boolean; { serializes staged sector-zero writes } cachedMBR : pointer; { cached mbr data, nil if not yet read } isBootDevice : boolean; { true if this is the device GRUB booted from } + Stats : TStorageStats; { cold tail: wrapping counters, timer ticks } end; { Filesystem driver descriptor } diff --git a/src/driver/storage/fs/fat32/driver.storage.fs.fat32.core.pas b/src/driver/storage/fs/fat32/driver.storage.fs.fat32.core.pas index 01ccea54..24742ac5 100644 --- a/src/driver/storage/fs/fat32/driver.storage.fs.fat32.core.pas +++ b/src/driver/storage/fs/fat32/driver.storage.fs.fat32.core.pas @@ -58,6 +58,25 @@ var sioc_allocVol : PStorage_Volume = nil; fatCtxListHead : PFATVolumeCtxNode = nil; +{ Bounded set operations; honour FAT_CACHE_WAYS rather than assuming two ways. } +procedure cacheTouch(cache : PFATCache; base, slot : uint32); +var w : uint32; +begin + for w := 0 to FAT_CACHE_WAYS-1 do + if cache^.Lines[base+w].LRU < FAT_CACHE_WAYS-1 then Inc(cache^.Lines[base+w].LRU); + cache^.Lines[slot].LRU := 0; +end; +function cacheVictim(cache : PFATCache; base : uint32) : uint32; +var w, victim : uint32; +begin + victim := base; + for w := 0 to FAT_CACHE_WAYS-1 do begin + if not cache^.Lines[base+w].Valid then begin cacheVictim := base+w; exit; end; + if cache^.Lines[base+w].LRU > cache^.Lines[victim].LRU then victim := base+w; + end; + cacheVictim := victim; +end; + function readFat(volume : PStorage_Volume; cluster : uint32; bootRecord : PBootRecord) : uint32; forward; procedure writeFat(volume : PStorage_Volume; cluster : uint32; value : uint32; bootRecord : PBootRecord); forward; procedure fatFreeClusterChain(volume : PStorage_Volume; info : PFATVolumeInfo; startCluster : uint32); forward; @@ -771,6 +790,7 @@ begin while true do begin asm pushf; cli end; if not cache^.Busy then begin + Inc(cache^.Device^.Stats.FatCacheMisses); cache^.Busy := true; asm popf end; break; @@ -846,10 +866,10 @@ begin asm pushf; cli end; for w := 0 to FAT_CACHE_WAYS - 1 do if cache^.Lines[baseSlot + w].Valid and (cache^.Lines[baseSlot + w].SectorIdx = fatSecIdx) then begin + Inc(cache^.Device^.Stats.FatCacheHits); dataPtr := puint32(uint32(cache^.Data) + ((baseSlot + w) * 512)); readFat := dataPtr[entryOff] and $0FFFFFFF; - cache^.Lines[baseSlot + w].LRU := 0; - cache^.Lines[baseSlot + (1 - w)].LRU := 1; + cacheTouch(cache,baseSlot,baseSlot+w); asm popf end; exit; end; @@ -860,23 +880,16 @@ begin if not cache^.Busy then begin for w := 0 to FAT_CACHE_WAYS - 1 do if cache^.Lines[baseSlot + w].Valid and (cache^.Lines[baseSlot + w].SectorIdx = fatSecIdx) then begin + Inc(cache^.Device^.Stats.FatCacheHits); dataPtr := puint32(uint32(cache^.Data) + ((baseSlot + w) * 512)); readFat := dataPtr[entryOff] and $0FFFFFFF; - cache^.Lines[baseSlot + w].LRU := 0; - cache^.Lines[baseSlot + (1 - w)].LRU := 1; + cacheTouch(cache,baseSlot,baseSlot+w); asm popf end; exit; end; + Inc(cache^.Device^.Stats.FatCacheMisses); cache^.Busy := true; - victim := baseSlot; - if not cache^.Lines[baseSlot].Valid then - victim := baseSlot - else if not cache^.Lines[baseSlot + 1].Valid then - victim := baseSlot + 1 - else if cache^.Lines[baseSlot + 1].LRU >= cache^.Lines[baseSlot].LRU then - victim := baseSlot + 1 - else - victim := baseSlot; + victim := cacheVictim(cache,baseSlot); dataPtr := puint32(uint32(cache^.Data) + (victim * 512)); doEvict := cache^.Lines[victim].Valid and cache^.Lines[victim].Dirty; if doEvict then begin @@ -900,11 +913,7 @@ begin cache^.Lines[victim].SectorIdx := fatSecIdx; cache^.Lines[victim].Valid := true; cache^.Lines[victim].Dirty := false; - cache^.Lines[victim].LRU := 0; - if victim = baseSlot then - cache^.Lines[baseSlot + 1].LRU := 1 - else - cache^.Lines[baseSlot].LRU := 1; + cacheTouch(cache,baseSlot,victim); readFat := dataPtr[entryOff] and $0FFFFFFF; cache^.Busy := false; asm popf end; @@ -936,14 +945,14 @@ begin asm pushf; cli end; for w := 0 to FAT_CACHE_WAYS - 1 do if cache^.Lines[baseSlot + w].Valid and (cache^.Lines[baseSlot + w].SectorIdx = fatSecIdx) then begin + Inc(cache^.Device^.Stats.FatCacheHits); dataPtr := puint32(uint32(cache^.Data) + ((baseSlot + w) * 512)); if value = 0 then dataPtr[entryOff] := 0 else dataPtr[entryOff] := (dataPtr[entryOff] and $F0000000) or (value and $0FFFFFFF); cache^.Lines[baseSlot + w].Dirty := true; - cache^.Lines[baseSlot + w].LRU := 0; - cache^.Lines[baseSlot + (1 - w)].LRU := 1; + cacheTouch(cache,baseSlot,baseSlot+w); asm popf end; exit; end; @@ -954,27 +963,20 @@ begin if not cache^.Busy then begin for w := 0 to FAT_CACHE_WAYS - 1 do if cache^.Lines[baseSlot + w].Valid and (cache^.Lines[baseSlot + w].SectorIdx = fatSecIdx) then begin + Inc(cache^.Device^.Stats.FatCacheHits); dataPtr := puint32(uint32(cache^.Data) + ((baseSlot + w) * 512)); if value = 0 then dataPtr[entryOff] := 0 else dataPtr[entryOff] := (dataPtr[entryOff] and $F0000000) or (value and $0FFFFFFF); cache^.Lines[baseSlot + w].Dirty := true; - cache^.Lines[baseSlot + w].LRU := 0; - cache^.Lines[baseSlot + (1 - w)].LRU := 1; + cacheTouch(cache,baseSlot,baseSlot+w); asm popf end; exit; end; + Inc(cache^.Device^.Stats.FatCacheMisses); cache^.Busy := true; - victim := baseSlot; - if not cache^.Lines[baseSlot].Valid then - victim := baseSlot - else if not cache^.Lines[baseSlot + 1].Valid then - victim := baseSlot + 1 - else if cache^.Lines[baseSlot + 1].LRU >= cache^.Lines[baseSlot].LRU then - victim := baseSlot + 1 - else - victim := baseSlot; + victim := cacheVictim(cache,baseSlot); dataPtr := puint32(uint32(cache^.Data) + (victim * 512)); doEvict := cache^.Lines[victim].Valid and cache^.Lines[victim].Dirty; if doEvict then begin @@ -1003,11 +1005,7 @@ begin else dataPtr[entryOff] := (dataPtr[entryOff] and $F0000000) or (value and $0FFFFFFF); cache^.Lines[victim].Dirty := true; - cache^.Lines[victim].LRU := 0; - if victim = baseSlot then - cache^.Lines[baseSlot + 1].LRU := 1 - else - cache^.Lines[baseSlot].LRU := 1; + cacheTouch(cache,baseSlot,victim); cache^.Busy := false; asm popf end; end; @@ -1064,6 +1062,7 @@ begin end; end; if not runMapEnsureCapacity(runMap, runMap.Count + 1) then exit; + runMap.Runs[runMap.Count].FileClusterStart := runMap.TotalClusters; runMap.Runs[runMap.Count].StartCluster := startCluster; runMap.Runs[runMap.Count].ClusterCount := clusterCount; runMap.Count := runMap.Count + 1; @@ -1609,44 +1608,43 @@ function FAT32FindRunForOffset(ofi : PFATOpenFile; fileOffset : uint32; var outRunIdx : uint32; var outRunFileCluster : uint32; var outStartCluster : uint32; var outClusterOffset : uint32; var outAvailableBytes : uint32) : boolean; -var - targetCluster : uint32; - clusterBase : uint32; - i : uint32; - bytesPerCluster : uint32; - byteOffsetInCl : uint32; +var target, i, lo, hi, mid, base, bpc, flags : uint32; begin FAT32FindRunForOffset := false; - if (ofi = nil) or (ofi^.VolumeInfo = nil) or (ofi^.RunMap.Count = 0) then - exit; - - bytesPerCluster := ofi^.VolumeInfo^.BytesPerCluster; - if bytesPerCluster = 0 then exit; - - targetCluster := fileOffset div bytesPerCluster; - byteOffsetInCl := fileOffset mod bytesPerCluster; - - if hintValid and (hintRunIdx < ofi^.RunMap.Count) and (hintFileCluster <= targetCluster) then begin - i := hintRunIdx; - clusterBase := hintFileCluster; - end else begin - i := 0; - clusterBase := 0; - end; - - while i < ofi^.RunMap.Count do begin - if targetCluster < (clusterBase + ofi^.RunMap.Runs[i].ClusterCount) then begin - outRunIdx := i; - outRunFileCluster := clusterBase; - outClusterOffset := targetCluster - clusterBase; - outStartCluster := ofi^.RunMap.Runs[i].StartCluster + outClusterOffset; - outAvailableBytes := ((ofi^.RunMap.Runs[i].ClusterCount - outClusterOffset) * bytesPerCluster) - byteOffsetInCl; - FAT32FindRunForOffset := true; - exit; + if (ofi = nil) or (ofi^.VolumeInfo = nil) or (ofi^.RunMap.Count = 0) then exit; + bpc := ofi^.VolumeInfo^.BytesPerCluster; + if bpc = 0 then exit; + target := fileOffset div bpc; + if target >= ofi^.RunMap.TotalClusters then exit; + { Only the index is shared: derive the base from the current run map. } + i := ofi^.Cursor.RunIdx; + if hintValid then i := hintRunIdx; + if i >= ofi^.RunMap.Count then i := 0; + base := ofi^.RunMap.Runs[i].FileClusterStart; + if (target < base) or (target-base >= ofi^.RunMap.Runs[i].ClusterCount) then begin + { Sequential boundary crossing stays O(1); random seeks are O(log runs). } + if (i+1 < ofi^.RunMap.Count) and + (target = ofi^.RunMap.Runs[i+1].FileClusterStart) then Inc(i) + else begin + lo := 0; hi := ofi^.RunMap.Count; + while lo < hi do begin + mid := lo + (hi-lo) div 2; + if ofi^.RunMap.Runs[mid].FileClusterStart <= target then lo := mid+1 + else hi := mid; + end; + if lo = 0 then exit; + i := lo-1; end; - clusterBase := clusterBase + ofi^.RunMap.Runs[i].ClusterCount; - i := i + 1; + base := ofi^.RunMap.Runs[i].FileClusterStart; end; + outRunIdx := i; outRunFileCluster := base; + outClusterOffset := target-base; + outStartCluster := ofi^.RunMap.Runs[i].StartCluster + outClusterOffset; + outAvailableBytes := (ofi^.RunMap.Runs[i].ClusterCount-outClusterOffset)*bpc - (fileOffset mod bpc); + asm pushfd; cli; pop flags end; + ofi^.Cursor.RunIdx := i; ofi^.Cursor.FileClusterStart := base; ofi^.Cursor.Valid := true; + if (flags and $200) <> 0 then asm sti end; + FAT32FindRunForOffset := true; end; function FAT32EnsureCapacityForWrite(ofi : PFATOpenFile; offset : uint32; byteCount : uint32) : TError; diff --git a/src/driver/storage/fs/fat32/driver.storage.fs.fat32.transfer.pas b/src/driver/storage/fs/fat32/driver.storage.fs.fat32.transfer.pas index a164c2fb..d82e6b88 100644 --- a/src/driver/storage/fs/fat32/driver.storage.fs.fat32.transfer.pas +++ b/src/driver/storage/fs/fat32/driver.storage.fs.fat32.transfer.pas @@ -19,6 +19,7 @@ procedure FAT32WriteFileAtOffsetAsync(volume : PStorage_Volume; directory : pcha implementation uses + proc.mgr, proc.types, driver.storage.mgr, core.util, arch.x86.util, memory.heap, @@ -57,6 +58,7 @@ var chunkBytes : uint32; chunkSectors : uint32; sectorLBA : uint32; + err : TError; begin transferSync := 0; if (ofi = nil) or (buffer = nil) or (byteCount = 0) then exit; @@ -79,21 +81,24 @@ begin hintRunIdx := runIdx; hintFileCluster := runFileCluster; - runByteOffset := currentOffset - (runFileCluster * bytesPerCluster); + runByteOffset := currentOffset mod bytesPerCluster; sectorOffset := runByteOffset mod sectorSize; sectorLBA := FAT32ClusterToLBA(ofi^.VolumeInfo, startCluster) + (runByteOffset div sectorSize); - if (sectorOffset <> 0) or (remaining < sectorSize) then begin - if scratch = nil then break; + if (sectorOffset <> 0) or (remaining < sectorSize) or + (((uint32(buffer)+bytesDone) and 1) <> 0) then begin + if (scratch = nil) or (scratchSize < sectorSize) then break; chunkBytes := sectorSize - sectorOffset; if chunkBytes > remaining then chunkBytes := remaining; - driver.storage.mgr.storage_read(ofi^.Volume^.device, sectorLBA, 1, scratch); + err := driver.storage.mgr.storage_read(ofi^.Volume^.device, sectorLBA, 1, scratch); + if err <> eNone then break; if mode = ftmRead then memcpy(uint32(scratch) + sectorOffset, uint32(buffer) + bytesDone, chunkBytes) else begin memcpy(uint32(buffer) + bytesDone, uint32(scratch) + sectorOffset, chunkBytes); - driver.storage.mgr.storage_write(ofi^.Volume^.device, sectorLBA, 1, scratch); + err := driver.storage.mgr.storage_write(ofi^.Volume^.device, sectorLBA, 1, scratch); + if err <> eNone then break; end; end else begin chunkBytes := remaining; @@ -106,13 +111,14 @@ begin chunkSectors := FAT_MAX_IO_SECTORS; chunkBytes := chunkSectors * sectorSize; if mode = ftmRead then - driver.storage.mgr.storage_read(ofi^.Volume^.device, sectorLBA, chunkSectors, + err := driver.storage.mgr.storage_read(ofi^.Volume^.device, sectorLBA, chunkSectors, puint32(uint32(buffer) + bytesDone)) else - driver.storage.mgr.storage_write(ofi^.Volume^.device, sectorLBA, chunkSectors, + err := driver.storage.mgr.storage_write(ofi^.Volume^.device, sectorLBA, chunkSectors, puint32(uint32(buffer) + bytesDone)); end; + if err <> eNone then break; bytesDone := bytesDone + chunkBytes; remaining := remaining - chunkBytes; if mode = ftmWrite then @@ -123,13 +129,18 @@ begin end; procedure asyncComplete(ctx : PFATTransferCtx; error : TError); +var callback : TIOCallback; data : pointer; begin if ctx = nil then exit; + if ctx^.Pumping then begin + ctx^.Finished := true; ctx^.LastError := error; exit; + end; if ctx^.BytesOut <> nil then ctx^.BytesOut^ := ctx^.BytesDone; - if ctx^.Callback <> nil then - ctx^.Callback(error, ctx^.CallbackData); + callback := ctx^.Callback; data := ctx^.CallbackData; + { Return the context before user code can close the volume or submit again. } FAT32TransferFree(ctx); + if callback <> nil then callback(error,data); end; procedure asyncContinue(ctx : PFATTransferCtx); forward; @@ -178,7 +189,64 @@ begin end; end; -procedure asyncContinue(ctx : PFATTransferCtx); +{ A batch owns four disjoint destination slices. On failure, drain every + accepted DMA before releasing context and report only the successful prefix. } +procedure readBatchFinish(ctx : PFATTransferCtx); +var i : uint32; err : TError; +begin + err := eNone; + if ctx^.ReadCount > 0 then for i := 0 to ctx^.ReadCount-1 do begin + if ctx^.ReadSlots[i].Error <> eNone then begin err := ctx^.ReadSlots[i].Error; break; end; + Inc(ctx^.BytesDone,ctx^.ReadSlots[i].Bytes); + end; + ctx^.ReadCount := 0; + if err <> eNone then asyncComplete(ctx,err) else asyncContinue(ctx); +end; + +procedure readSlotDone(error : TError; data : pointer); +var slot : PFATReadSlot; ctx : PFATTransferCtx; +begin + slot := PFATReadSlot(data); ctx := slot^.Owner; + slot^.Error := error; Dec(ctx^.ReadPending); + if (ctx^.ReadPending = 0) and not ctx^.ReadSubmitting then readBatchFinish(ctx); +end; + +procedure readBatch(ctx : PFATTransferCtx); +var pos, remaining, idx, base, cluster, off, available, sectors, bytes, lba, size, depth : uint32; + slot : PFATReadSlot; err : TError; +begin + ctx^.ReadSubmitting := true; ctx^.ReadCount := 0; ctx^.ReadPending := 0; + pos := ctx^.BytesDone; size := ctx^.OpenFile^.VolumeInfo^.BootRecord.sectorSize; + depth := ctx^.Volume^.device^.maxActive; + if depth > 4 then depth := 4; + if depth = 0 then depth := 1; + while (ctx^.ReadCount < depth) and (pos < ctx^.ByteCount) do begin + remaining := ctx^.ByteCount-pos; + if remaining < size then break; + slot := @ctx^.ReadSlots[ctx^.ReadCount]; + slot^.Owner := ctx; slot^.Bytes := 0; slot^.Error := eNone; + Inc(ctx^.ReadCount); + if not FAT32FindRunForOffset(ctx^.OpenFile,ctx^.Offset+pos,false,0,0, + idx,base,cluster,off,available) then begin slot^.Error := eCorruptFilesystem; break; end; + bytes := remaining; if bytes > available then bytes := available; + sectors := bytes div size; + if sectors > FAT_MAX_IO_SECTORS then sectors := FAT_MAX_IO_SECTORS; + if sectors = 0 then begin slot^.Error := eInvalidArgument; break; end; + bytes := sectors*size; slot^.Bytes := bytes; + lba := FAT32ClusterToLBA(ctx^.OpenFile^.VolumeInfo,cluster) + + ((ctx^.Offset+pos) mod ctx^.OpenFile^.VolumeInfo^.BytesPerCluster) div size; + Inc(ctx^.ReadPending); + err := driver.storage.mgr.storage_read_async(ctx^.Volume^.device,lba,sectors, + puint32(uint32(ctx^.Buffer)+pos),@readSlotDone,slot); + if err <> eNone then begin Dec(ctx^.ReadPending); slot^.Error := err; break; end; + if slot^.Error <> eNone then break; { synchronous completion failure } + Inc(pos,bytes); + end; + ctx^.ReadSubmitting := false; + if ctx^.ReadPending = 0 then readBatchFinish(ctx); +end; + +procedure asyncAdvance(ctx : PFATTransferCtx); var runIdx : uint32; runFileCluster : uint32; @@ -216,12 +284,13 @@ begin ctx^.RunHintIndex := runIdx; ctx^.RunHintFileCluster := runFileCluster; - runByteOffset := currentOffset - (runFileCluster * bytesPerCluster); + runByteOffset := currentOffset mod bytesPerCluster; sectorOffset := runByteOffset mod sectorSize; ctx^.PendingLBA := FAT32ClusterToLBA(ctx^.OpenFile^.VolumeInfo, startCluster) + (runByteOffset div sectorSize); ctx^.PendingBufferOffset := ctx^.BytesDone; - if (sectorOffset <> 0) or (remaining < sectorSize) then begin + if (sectorOffset <> 0) or (remaining < sectorSize) or + (((uint32(ctx^.Buffer)+ctx^.BytesDone) and 1) <> 0) then begin chunkBytes := sectorSize - sectorOffset; if chunkBytes > remaining then chunkBytes := remaining; @@ -235,6 +304,8 @@ begin exit; end; + if ctx^.Mode = ftmRead then begin readBatch(ctx); exit; end; + chunkBytes := remaining; if chunkBytes > availableBytes then chunkBytes := availableBytes; @@ -260,6 +331,36 @@ begin asyncComplete(ctx, err); end; +{ Serialize pump state against IRQ completion and turn inline completions + into iteration, so fake/RAM drivers cannot recursively exhaust the stack. } +procedure asyncContinue(ctx : PFATTransferCtx); +var flags : uint32; +begin + asm pushfd; cli; pop flags end; + if ctx^.Pumping then ctx^.Resume := true + else begin + ctx^.Pumping := true; + repeat + ctx^.Resume := false; + asyncAdvance(ctx); + until (not ctx^.Resume) or ctx^.Finished; + ctx^.Pumping := false; + if ctx^.Finished then asyncComplete(ctx,ctx^.LastError); + end; + if (flags and $200) <> 0 then asm sti end; +end; + +type + PReadWait = ^TReadWait; + TReadWait = record Done, Bytes : uint32; Process : PProcessContext; end; +procedure readWaitDone(error : TError; data : pointer); +var wait : PReadWait; +begin + wait := PReadWait(data); + if wait^.Process <> nil then wait^.Process^.State := psReady; + puint32(@wait^.Done)^ := 1; +end; + function FAT32ReadFileAtOffset(volume : PStorage_Volume; directory : pchar; fileName : pchar; offset : uint32; buffer : puint32; byteCount : uint32; ctx : pointer) : uint32; var @@ -267,6 +368,7 @@ var tempCtx : pointer; fileSize : uint32; readLimit : uint32; + wait : TReadWait; begin FAT32ReadFileAtOffset := 0; if (buffer = nil) or (byteCount = 0) then exit; @@ -280,9 +382,16 @@ begin if ofi = nil then exit; readLimit := FAT32GetReadLimit(ofi, offset, byteCount); - if readLimit > 0 then - FAT32ReadFileAtOffset := transferSync(ofi, buffer, offset, readLimit, ftmRead, - ofi^.ScratchSector, ofi^.ScratchSize); + if readLimit > 0 then begin + wait.Done := 0; wait.Bytes := 0; wait.Process := CurrentProcess; + FAT32ReadFileAtOffsetAsync(volume,directory,fileName,offset,buffer,readLimit,ofi, + @wait.Bytes,@readWaitDone,@wait); + asm pushf; cli end; + if (puint32(@wait.Done)^ = 0) and (CurrentProcess <> nil) then CurrentProcess^.State := psAwaiting; + asm popf end; + while puint32(@wait.Done)^ = 0 do asm hlt end; + FAT32ReadFileAtOffset := wait.Bytes; + end; if tempCtx <> nil then FAT32CloseFile(tempCtx); diff --git a/src/driver/storage/fs/fat32/driver.storage.fs.fat32.types.pas b/src/driver/storage/fs/fat32/driver.storage.fs.fat32.types.pas index e4eef98c..6096cffb 100644 --- a/src/driver/storage/fs/fat32/driver.storage.fs.fat32.types.pas +++ b/src/driver/storage/fs/fat32/driver.storage.fs.fat32.types.pas @@ -99,6 +99,7 @@ type TFATRun = record StartCluster : uint32; ClusterCount : uint32; + FileClusterStart : uint32; end; TFATAllocHint = record @@ -183,6 +184,13 @@ type PFATVolumeCtx = ^TFATVolumeCtx; PFATTransferCtx = ^TFATTransferCtx; + PFATReadSlot = ^TFATReadSlot; + TFATReadSlot = record + Owner : PFATTransferCtx; + Bytes : uint32; + Error : TError; + end; + TFATTransferCtx = record Next : PFATTransferCtx; VolumeCtx : PFATVolumeCtx; @@ -209,6 +217,10 @@ type RunHintIndex : uint32; RunHintFileCluster : uint32; LastError : TError; + Pumping, Resume, Finished : boolean; + ReadSubmitting : boolean; + ReadPending, ReadCount : uint32; + ReadSlots : array[0..3] of TFATReadSlot; end; TFATVolumeCtx = record diff --git a/src/test/test.pas b/src/test/test.pas index fece7d8b..b9ddd739 100644 --- a/src/test/test.pas +++ b/src/test/test.pas @@ -34,6 +34,7 @@ uses driver.storage.vfs, driver.storage.test, driver.storage.metatest, + driver.storage.perftest, driver.storage.ctl.ahci, core.enc.fnv1a, core.enc.djb2, @@ -71,6 +72,7 @@ begin driver.storage.vfs.UnitTest; driver.storage.test.UnitTest; driver.storage.metatest.UnitTest; + driver.storage.perftest.UnitTest; driver.storage.ctl.ahci.UnitTest; core.enc.fnv1a.UnitTest; core.enc.djb2.UnitTest;