Improve storage performance with read pipelining, run lookup, request merging, and counters

doc/src/driver/storage/driver.storage.mgr.md

@@ -219,3 +219,37 @@ Drop-in procedure replacements for callers using the old void-returning API. For
 - `dispatch_next` is an internal procedure called with interrupts disabled. It dequeues the head of the CFIFO, allocates a heap copy via `ioreq_copy` (for ISR safety), and invokes the appropriate `dispatchRead` or `dispatchWrite` on the device. If neither dispatch pointer is set, the request is failed with `eDeviceNotFound`.
 - The blocking path relies on pointer-dereferenced spin (`puint32(@wait.Done)^`) to prevent FPC from caching the `Done` flag in a register across the `hlt` instruction.
 - Devices without `dispatchRead`/`dispatchWrite` set (i.e., drivers not yet migrated to Phase 2) will have `submit_io` return `eDeviceNotFound`. Legacy drivers must be migrated before they can use the new I/O path.
+
+
+## Bounded performance path
+
+The manager merges up to 32 consecutive queued requests when operation, device,
+LBA range, and virtual buffer range are contiguous. It never searches past the
+queue head or delays a lone request to create a merge. Drivers opt in through
+`mergeMaxSectors`; SATA AHCI advertises 65,536 sectors, while other drivers default
+to zero (disabled). A merged command delivers one completion to each original
+request, in FIFO order. A command error fails every member conservatively.
+
+`storage_get_stats` takes an interrupt-protected snapshot of the device counters.
+`DISK info <index>` displays accepted/completed/failed logical requests, hardware
+commands, merged requests, queue-full rejections, queue and active high-water
+marks, successful read/write sectors, request-size buckets, queue/service ticks,
+and FAT metadata cache hits/misses. Queue ticks are accumulated per logical
+request; service ticks are accumulated per hardware command. Tick values use
+`Counters.c32`, not milliseconds. Counters are unsigned 32-bit values, start at
+device registration, and wrap; use snapshot differences over bounded benchmark
+intervals. Size buckets are <=8, 9..256, and >256 sectors.
+
+FAT32 blocking and async reads share a pooled transfer engine with batches of up
+to four outstanding reads, capped by device queue depth. Destination slices are
+disjoint. On any error, the engine drains every accepted request before releasing
+its context and reports only the contiguous successful byte prefix. Unaligned
+edges and odd DMA addresses use the context's private scratch sector. Writes
+remain sequential. Run maps store cumulative file-cluster offsets; an open-file
+cursor handles sequential locality and binary search handles random seeks.
+
+`STORPERF` runs memory-backed regression checks (also included in boot unit tests)
+for reverse completion order, failures, unaligned reads, run lookup, queue
+rejection, merging limits, and callback accounting. It does not access real disks
+or measure hardware throughput. Compare counters alongside `STORBENCH` and file
+benchmarks on contiguous and fragmented files for actual performance validation.

src/app/app.diskcmd.pas

@@ -150,6 +150,7 @@ var
     device   : PStorage_Device;
     volCount : uint32;
     freeSec  : uint32;
+    stats : TStorageStats;
 begin
     if paramCount(params) < 2 then begin
         println('Usage: DISK info <disk>');
@@ -191,6 +192,16 @@ begin
     else
         volCount := 0;
     print('Partitions:      '); printint(volCount); println('');
+    storage_get_stats(device,stats);
+    print('I/O submitted/completed/failed: '); printint(stats.Submitted); print(' / '); printint(stats.Completed); print(' / '); printint(stats.Failed); println('');
+    print('Commands / merged requests: '); printint(stats.Commands); print(' / '); printint(stats.Merged); println('');
+    print('Queue peak / active peak / queue full: '); printint(stats.QueuePeak); print(' / '); printint(stats.ActivePeak); print(' / '); printint(stats.QueueFull); println('');
+    print('Read / written sectors: '); printint(stats.ReadSectors); print(' / '); printint(stats.WriteSectors); println('');
+    print('Requests <=8 / <=256 / >256 sectors: '); printint(stats.SmallRequests); print(' / '); printint(stats.MediumRequests); print(' / '); printint(stats.LargeRequests); println('');
+    print('Queue ticks total / max: '); printint(stats.QueueTicks); print(' / '); printint(stats.MaxQueueTicks); println('');
+    print('Service ticks total / max: '); printint(stats.ServiceTicks); print(' / '); printint(stats.MaxServiceTicks); println('');
+    print('FAT cache hits / misses: '); printint(stats.FatCacheHits); print(' / '); printint(stats.FatCacheMisses); println('');
+
 end;
 
 { ---------- DISK WIPE ---------- }

src/driver/storage/ctl/ahci/driver.storage.ctl.ahci.pas

@@ -397,6 +397,7 @@ begin
         storageDev^.controllerId0  := uint32(device);  { store PAHCI_Device for callback wrappers }
         storageDev^.sectorSize     := storageDev_sectorSize;
         storageDev^.maxSectorCount := sec_count;
+        if not isATAPI then storageDev^.mergeMaxSectors := 65536;
         storageDev^.start          := 0;
         storageDev^.volumes        := DL_New(sizeof(Pointer));
 

src/driver/storage/driver.storage.iorequest.pas

@@ -39,8 +39,8 @@ function  ioreq_alloc(reqType  : TIORequestType;
 procedure ioreq_free(req : PIORequest);
 
 { Copy an existing TIORequest into a new pooled allocation (deep copy of the
-  record, NOT the buffer it points to). Used by dispatch_next to create an
+  record, NOT the buffer it points to). Used by submit_io before enqueueing
-  ISR-safe copy from the CFIFO value-element. }
+  the owned pointer. Manager-only merge/timing fields are initialised here. }
 function  ioreq_copy(src : PIORequest) : PIORequest;
 
 implementation
@@ -132,6 +132,9 @@ begin
         ioreq_alloc := nil;
         exit;
     end;
+    req^.MergeNext := nil;
+    req^.OriginalSectors := sectors;
+    req^.QueuedTick := 0; req^.DispatchTick := 0;
     req^.RequestType := reqType;
     req^.State       := iosPending;
     req^.Device      := device;
@@ -161,6 +164,9 @@ begin
         ioreq_copy := nil;
         exit;
     end;
+    dst^.MergeNext := nil;
+    dst^.OriginalSectors := src^.SectorCount;
+    dst^.QueuedTick := 0; dst^.DispatchTick := 0;
     dst^.RequestType := src^.RequestType;
     dst^.State       := src^.State;
     dst^.Device      := src^.Device;

src/driver/storage/driver.storage.types.pas

@@ -252,11 +252,22 @@ type
         UserData     : pointer;         { points back to caller's stack-local request (blocking path) }
         Callback     : TIOCallback;     { async completion callback (nil for blocking path) }
         CallbackData : pointer;         { opaque data passed to Callback }
+        { Manager-owned fields; callers need not initialise these. }
+        MergeNext : PIORequest;
+        OriginalSectors, QueuedTick, DispatchTick : uint32;
     end;
 
     byteArray8  = array[0..7] of char;
     PByteArray8 = ^byteArray8;
 
+    TStorageStats = record
+        Submitted, Completed, Failed, Commands, Merged, QueueFull : uint32;
+        ReadSectors, WriteSectors, QueuePeak, ActivePeak : uint32;
+        QueueTicks, ServiceTicks, MaxQueueTicks, MaxServiceTicks : uint32;
+        SmallRequests, MediumRequests, LargeRequests : uint32;
+        FatCacheHits, FatCacheMisses : uint32;
+    end;
+
     { Generic storage device }
     TStorage_Device = record
         id               : uint8;
@@ -272,6 +283,7 @@ type
         requestQueue     : PCFIFOQueue;       { per-device I/O request CFIFO }
         dispatching      : boolean; { prevents recursive synchronous-error queue draining }
         activeCount      : uint32;            { number of commands currently dispatched to HW }
+        mergeMaxSectors  : uint32; { zero disables merging; driver sets safe command limit }
         maxActive        : uint32;            { max concurrent commands (NCQ depth) }
 
         maxSectorCount   : uint32;
@@ -281,6 +293,7 @@ type
         mbrWriteBusy     : boolean; { serializes staged sector-zero writes }
         cachedMBR        : pointer;           { cached mbr data, nil if not yet read }
         isBootDevice     : boolean;           { true if this is the device GRUB booted from }
+        Stats            : TStorageStats; { cold tail: wrapping counters, timer ticks }
     end;
 
     { Filesystem driver descriptor }

src/driver/storage/fs/fat32/driver.storage.fs.fat32.types.pas

@@ -99,6 +99,7 @@ type
     TFATRun = record
         StartCluster : uint32;
         ClusterCount : uint32;
+        FileClusterStart : uint32;
     end;
 
     TFATAllocHint = record
@@ -183,6 +184,13 @@ type
     PFATVolumeCtx = ^TFATVolumeCtx;
     PFATTransferCtx = ^TFATTransferCtx;
 
+    PFATReadSlot = ^TFATReadSlot;
+    TFATReadSlot = record
+        Owner : PFATTransferCtx;
+        Bytes : uint32;
+        Error : TError;
+    end;
+
     TFATTransferCtx = record
         Next                 : PFATTransferCtx;
         VolumeCtx            : PFATVolumeCtx;
@@ -209,6 +217,10 @@ type
         RunHintIndex         : uint32;
         RunHintFileCluster   : uint32;
         LastError            : TError;
+        Pumping, Resume, Finished : boolean;
+        ReadSubmitting : boolean;
+        ReadPending, ReadCount : uint32;
+        ReadSlots : array[0..3] of TFATReadSlot;
     end;
 
     TFATVolumeCtx = record

src/test/test.pas

@@ -34,6 +34,7 @@ uses
     driver.storage.vfs,
     driver.storage.test,
     driver.storage.metatest,
+    driver.storage.perftest,
     driver.storage.ctl.ahci,
     core.enc.fnv1a,
     core.enc.djb2,
@@ -71,6 +72,7 @@ begin
     driver.storage.vfs.UnitTest;
     driver.storage.test.UnitTest;
     driver.storage.metatest.UnitTest;
+    driver.storage.perftest.UnitTest;
     driver.storage.ctl.ahci.UnitTest;
     core.enc.fnv1a.UnitTest;
     core.enc.djb2.UnitTest;