Improve storage performance with read pipelining, run lookup, request merging, and counters

src/app/app.diskcmd.pas

@@ -150,6 +150,7 @@ var
     device   : PStorage_Device;
     volCount : uint32;
     freeSec  : uint32;
+    stats : TStorageStats;
 begin
     if paramCount(params) < 2 then begin
         println('Usage: DISK info <disk>');
@@ -191,6 +192,16 @@ begin
     else
         volCount := 0;
     print('Partitions:      '); printint(volCount); println('');
+    storage_get_stats(device,stats);
+    print('I/O submitted/completed/failed: '); printint(stats.Submitted); print(' / '); printint(stats.Completed); print(' / '); printint(stats.Failed); println('');
+    print('Commands / merged requests: '); printint(stats.Commands); print(' / '); printint(stats.Merged); println('');
+    print('Queue peak / active peak / queue full: '); printint(stats.QueuePeak); print(' / '); printint(stats.ActivePeak); print(' / '); printint(stats.QueueFull); println('');
+    print('Read / written sectors: '); printint(stats.ReadSectors); print(' / '); printint(stats.WriteSectors); println('');
+    print('Requests <=8 / <=256 / >256 sectors: '); printint(stats.SmallRequests); print(' / '); printint(stats.MediumRequests); print(' / '); printint(stats.LargeRequests); println('');
+    print('Queue ticks total / max: '); printint(stats.QueueTicks); print(' / '); printint(stats.MaxQueueTicks); println('');
+    print('Service ticks total / max: '); printint(stats.ServiceTicks); print(' / '); printint(stats.MaxServiceTicks); println('');
+    print('FAT cache hits / misses: '); printint(stats.FatCacheHits); print(' / '); printint(stats.FatCacheMisses); println('');
+
 end;
 
 { ---------- DISK WIPE ---------- }

src/driver/storage/ctl/ahci/driver.storage.ctl.ahci.pas

@@ -397,6 +397,7 @@ begin
         storageDev^.controllerId0  := uint32(device);  { store PAHCI_Device for callback wrappers }
         storageDev^.sectorSize     := storageDev_sectorSize;
         storageDev^.maxSectorCount := sec_count;
+        if not isATAPI then storageDev^.mergeMaxSectors := 65536;
         storageDev^.start          := 0;
         storageDev^.volumes        := DL_New(sizeof(Pointer));
 

src/driver/storage/driver.storage.iorequest.pas

@@ -39,8 +39,8 @@ function  ioreq_alloc(reqType  : TIORequestType;
 procedure ioreq_free(req : PIORequest);
 
 { Copy an existing TIORequest into a new pooled allocation (deep copy of the
-  record, NOT the buffer it points to). Used by dispatch_next to create an
-  ISR-safe copy from the CFIFO value-element. }
+  record, NOT the buffer it points to). Used by submit_io before enqueueing
+  the owned pointer. Manager-only merge/timing fields are initialised here. }
 function  ioreq_copy(src : PIORequest) : PIORequest;
 
 implementation
@@ -132,6 +132,9 @@ begin
         ioreq_alloc := nil;
         exit;
     end;
+    req^.MergeNext := nil;
+    req^.OriginalSectors := sectors;
+    req^.QueuedTick := 0; req^.DispatchTick := 0;
     req^.RequestType := reqType;
     req^.State       := iosPending;
     req^.Device      := device;
@@ -161,6 +164,9 @@ begin
         ioreq_copy := nil;
         exit;
     end;
+    dst^.MergeNext := nil;
+    dst^.OriginalSectors := src^.SectorCount;
+    dst^.QueuedTick := 0; dst^.DispatchTick := 0;
     dst^.RequestType := src^.RequestType;
     dst^.State       := src^.State;
     dst^.Device      := src^.Device;

src/driver/storage/driver.storage.types.pas

@@ -252,11 +252,22 @@ type
         UserData     : pointer;         { points back to caller's stack-local request (blocking path) }
         Callback     : TIOCallback;     { async completion callback (nil for blocking path) }
         CallbackData : pointer;         { opaque data passed to Callback }
+        { Manager-owned fields; callers need not initialise these. }
+        MergeNext : PIORequest;
+        OriginalSectors, QueuedTick, DispatchTick : uint32;
     end;
 
     byteArray8  = array[0..7] of char;
     PByteArray8 = ^byteArray8;
 
+    TStorageStats = record
+        Submitted, Completed, Failed, Commands, Merged, QueueFull : uint32;
+        ReadSectors, WriteSectors, QueuePeak, ActivePeak : uint32;
+        QueueTicks, ServiceTicks, MaxQueueTicks, MaxServiceTicks : uint32;
+        SmallRequests, MediumRequests, LargeRequests : uint32;
+        FatCacheHits, FatCacheMisses : uint32;
+    end;
+
     { Generic storage device }
     TStorage_Device = record
         id               : uint8;
@@ -272,6 +283,7 @@ type
         requestQueue     : PCFIFOQueue;       { per-device I/O request CFIFO }
         dispatching      : boolean; { prevents recursive synchronous-error queue draining }
         activeCount      : uint32;            { number of commands currently dispatched to HW }
+        mergeMaxSectors  : uint32; { zero disables merging; driver sets safe command limit }
         maxActive        : uint32;            { max concurrent commands (NCQ depth) }
 
         maxSectorCount   : uint32;
@@ -281,6 +293,7 @@ type
         mbrWriteBusy     : boolean; { serializes staged sector-zero writes }
         cachedMBR        : pointer;           { cached mbr data, nil if not yet read }
         isBootDevice     : boolean;           { true if this is the device GRUB booted from }
+        Stats            : TStorageStats; { cold tail: wrapping counters, timer ticks }
     end;
 
     { Filesystem driver descriptor }

src/driver/storage/fs/fat32/driver.storage.fs.fat32.types.pas

@@ -99,6 +99,7 @@ type
     TFATRun = record
         StartCluster : uint32;
         ClusterCount : uint32;
+        FileClusterStart : uint32;
     end;
 
     TFATAllocHint = record
@@ -183,6 +184,13 @@ type
     PFATVolumeCtx = ^TFATVolumeCtx;
     PFATTransferCtx = ^TFATTransferCtx;
 
+    PFATReadSlot = ^TFATReadSlot;
+    TFATReadSlot = record
+        Owner : PFATTransferCtx;
+        Bytes : uint32;
+        Error : TError;
+    end;
+
     TFATTransferCtx = record
         Next                 : PFATTransferCtx;
         VolumeCtx            : PFATVolumeCtx;
@@ -209,6 +217,10 @@ type
         RunHintIndex         : uint32;
         RunHintFileCluster   : uint32;
         LastError            : TError;
+        Pumping, Resume, Finished : boolean;
+        ReadSubmitting : boolean;
+        ReadPending, ReadCount : uint32;
+        ReadSlots : array[0..3] of TFATReadSlot;
     end;
 
     TFATVolumeCtx = record

src/test/test.pas

@@ -34,6 +34,7 @@ uses
     driver.storage.vfs,
     driver.storage.test,
     driver.storage.metatest,
+    driver.storage.perftest,
     driver.storage.ctl.ahci,
     core.enc.fnv1a,
     core.enc.djb2,
@@ -71,6 +72,7 @@ begin
     driver.storage.vfs.UnitTest;
     driver.storage.test.UnitTest;
     driver.storage.metatest.UnitTest;
+    driver.storage.perftest.UnitTest;
     driver.storage.ctl.ahci.UnitTest;
     core.enc.fnv1a.UnitTest;
     core.enc.djb2.UnitTest;