VFS improvements, IO fixes, IO performance improvements.

src/app/app.mgr.pas

@@ -26,7 +26,7 @@ uses
     boot.mgr,
     debug.tracer, io.stdio, proc.mgr,
     //progs
-    app.base64, app.md5sum, app.dhclient, app.vbeinfo, app.testcmd, app.ping, app.meminfo, app.setres,
+    app.base64, app.md5sum, app.dhclient, app.vbeinfo, app.testcmd, app.ping, app.meminfo, app.setres, app.iotest,
     //drivers
     driver.storage.ctl.ram, driver.mgr,
     //network
@@ -92,6 +92,7 @@ begin
     app.bsod.init();
     app.vterminal.init;
     app.filebrowser.init();
+    app.iotest.init();
 end;
 
 Initialization

src/driver/storage/ctl/ahci/driver.storage.ctl.ahci.pas

@@ -450,11 +450,13 @@ begin
             storageDev^.writable   := false;
             storageDev^.dispatchRead       := TDriverDispatch(@ahci_dispatch_atapi_read);
             storageDev^.dispatchWrite      := TDriverDispatch(nil);
+            storageDev^.maxActive          := 1;
         end else begin
             storageDev^.controller := TControllerType.ControllerAHCI;
             storageDev^.writable   := true;
             storageDev^.dispatchRead       := TDriverDispatch(@ahci_dispatch_read);
             storageDev^.dispatchWrite      := TDriverDispatch(@ahci_dispatch_write);
+            storageDev^.maxActive          := 32;  { AHCI supports up to 32 command slots }
         end;
 
         driver.storage.mgr.register_device(storageDev);

src/driver/storage/ctl/driver.storage.ctl.ram.pas

@@ -82,33 +82,6 @@ end;
 
 { ---- Filesystem hook callbacks ---- }
 
-{ PPReadHook — load entire file into a new kalloc'd buffer.
-  Sets buffer^ to the data pointer, bytecount^ to the size.
-  Returns 0 on success, 1 on failure. }
-function rd_Read(volume : PStorage_Volume; directory : pchar;
-                 fileName : pchar; buffer : puint32;
-                 bytecount : puint32) : uint32;
-var
-    clean : pchar;
-    idx   : sint32;
-    dest  : puint8;
-begin
-    rd_Read := 1;
-    clean := stripLeadingSlash(fileName);
-    if clean = nil then exit;
-    idx := findFileByName(clean);
-    if idx < 0 then exit;
-
-    { VFS expects a fresh buffer it can own }
-    dest := puint8(kalloc(Files[idx].Size));
-    if dest = nil then exit;
-    memcpy(uint32(Files[idx].Data), uint32(dest), Files[idx].Size);
-
-    buffer^ := uint32(dest);
-    bytecount^ := Files[idx].Size;
-    rd_Read := 0;
-end;
-
 { PPReadOffsetHook — read byteCount bytes at offset into caller buffer.
   Returns actual bytes copied. }
 function rd_ReadOffset(volume : PStorage_Volume; directory : pchar;
@@ -133,6 +106,20 @@ begin
     rd_ReadOffset := copyLen;
 end;
 
+function rd_FileSize(volume : PStorage_Volume; directory : pchar;
+                     fileName : pchar) : uint32;
+var
+    clean : pchar;
+    idx   : sint32;
+begin
+    rd_FileSize := 0;
+    clean := stripLeadingSlash(fileName);
+    if clean = nil then exit;
+    idx := findFileByName(clean);
+    if idx < 0 then exit;
+    rd_FileSize := Files[idx].Size;
+end;
+
 { PPReadDirHook — return a linked list of TDirectory_Entry for all files.
   VFS owns the returned list and frees each entry^.fileName. }
 function rd_ReadDir(volume : PStorage_Volume; directory : pchar;
@@ -269,9 +256,9 @@ begin
     { Set up a minimal filesystem descriptor with our callbacks }
     memset(uint32(@RAMFS), 0, sizeof(TFilesystem));
     RAMFS.sName := 'ramfs';
-    RAMFS.readCallback := @rd_Read;
     RAMFS.readDirCallback := @rd_ReadDir;
     RAMFS.readOffsetCallback := @rd_ReadOffset;
+    RAMFS.fileSizeCallback := @rd_FileSize;
     RAMFS.identifyCallback := @rd_Identify;
 
     { Set up a virtual volume backed by our filesystem }

src/driver/storage/driver.storage.fdtable.pas

@@ -42,10 +42,7 @@ type
         FileName    : pchar;       { filename within that directory }
         VFSDir      : pchar;       { full VFS parent directory path, e.g. '/sys' }
         OpenMode    : uint8;       { TOpenMode ordinal — avoids VFS type dependency }
-        DataBuffer  : puint32;     { pre-loaded data for small files, nil otherwise }
+        DataSize    : uint32;      { cached file size in bytes (from fileSizeCallback) }
-        DataSize    : uint32;      { size of pre-loaded data in bytes }
-        Loaded      : boolean;     { true if pre-loaded into DataBuffer }
-        StreamOff   : uint32;      { current byte offset for streaming / on-demand reads }
         DeviceOps   : pointer;     { PVFSDeviceOps — non-nil for device FDs }
         DeviceData  : pointer;     { opaque data passed to device callbacks }
     end;
@@ -81,6 +78,10 @@ function  fd_close(table : PFDTable; handle : uint32) : boolean;
   Used during process cleanup. }
 procedure fd_close_all(table : PFDTable);
 
+{ Close all file descriptors in the table that reference a specific volume.
+  Called when a volume is removed to prevent dangling pointers. }
+procedure fd_close_for_volume(table : PFDTable; vol : PStorage_Volume);
+
 implementation
 
 function fd_table_new : PFDTable;
@@ -98,10 +99,6 @@ begin
     if entry = nil then exit;
     if not entry^.InUse then exit;
 
-    if entry^.DataBuffer <> nil then begin
-        kfree(entry^.DataBuffer);
-        entry^.DataBuffer := nil;
-    end;
     if entry^.Directory <> nil then begin
         kfree(void(entry^.Directory));
         entry^.Directory := nil;
@@ -116,9 +113,7 @@ begin
     end;
 
     entry^.InUse := false;
-    entry^.Loaded := false;
     entry^.DataSize := 0;
-    entry^.StreamOff := 0;
     entry^.DeviceOps := nil;
     entry^.DeviceData := nil;
 end;
@@ -174,4 +169,16 @@ begin
     end;
 end;
 
+procedure fd_close_for_volume(table : PFDTable; vol : PStorage_Volume);
+var
+    i : uint32;
+begin
+    if table = nil then exit;
+    if vol = nil then exit;
+    for i := 0 to MAX_FDS - 1 do begin
+        if table^.Entries[i].InUse and (table^.Entries[i].Volume = vol) then
+            fd_free_entry(@table^.Entries[i]);
+    end;
+end;
+
 end.

src/driver/storage/driver.storage.filedispatch.pas

@@ -219,10 +219,9 @@ begin
     validity := driver.storage.vfs.PathValid(absPath);
     if validity <> pvFile then exit;
 
-    { 2. Open the file in stream mode — only reads the bytes we ask for,
+    { 2. Open the file for reading — reads the magic bytes on demand. }
-         avoids pre-loading the entire file just to check a few magic bytes. }
     err := eNone;
-    fh := driver.storage.vfs.OpenFile(absPath, omStream, @err);
+    fh := driver.storage.vfs.OpenFile(absPath, omRead, @err);
     if fh = 0 then exit;
 
     memset(uint32(@headerBuf[0]), 0, MAX_MAGIC_LEN);

src/driver/storage/driver.storage.mgr.pas

@@ -185,7 +185,8 @@ begin
     { Initialise new Phase 2 fields }
     device^.removed := false;
     device^.requestQueue := CFIFO_New(SizeOf(TIORequest), 32);
-    device^.activeRequest := nil;
+    device^.activeCount := 0;
+    if device^.maxActive = 0 then device^.maxActive := 1;
     device^.cachedMBR := nil;
     device^.isBootDevice := false;
     { dispatchRead/dispatchWrite: left as-is — driver sets them before
@@ -343,8 +344,9 @@ var
     reqBuf  : TIORequest;
     heapReq : PIORequest;
 begin
-    if device^.activeRequest <> nil then exit;   { hardware busy }
+    { Dispatch queued requests up to maxActive hardware slots }
-    if not CFIFO_Dequeue(device^.requestQueue, @reqBuf) then exit;  { queue empty }
+    while device^.activeCount < device^.maxActive do begin
+        if not CFIFO_Dequeue(device^.requestQueue, @reqBuf) then break;  { queue empty }
 
         { Heap copy for ISR safety — driver completion may fire on any stack }
         heapReq := ioreq_copy(@reqBuf);
@@ -352,10 +354,10 @@ begin
         { Out of memory — notify caller via callback }
             if reqBuf.Callback <> nil then
                 reqBuf.Callback(eOutOfMemory, reqBuf.CallbackData);
-        exit;
+            break;
         end;
 
-    device^.activeRequest := heapReq;
+        device^.activeCount := device^.activeCount + 1;
         heapReq^.State := iosDispatched;
 
         case heapReq^.RequestType of
@@ -363,8 +365,6 @@ begin
                 if device^.dispatchRead <> nil then
                     device^.dispatchRead(device, heapReq)
                 else begin
-                { No new dispatch — shouldn't happen if submit_io chose this path,
-                  but handle defensively }
                     complete_io(heapReq, false, eDeviceNotFound);
                 end;
             end;
@@ -375,6 +375,7 @@ begin
                     complete_io(heapReq, false, eDeviceNotFound);
             end;
         end;
+    end;
 end;
 
 function submit_io(request : PIORequest) : TError;
@@ -472,9 +473,10 @@ begin
         request^.Error := error;
     end;
 
-    { Clear active slot on the device }
+    { Decrement active count on the device }
     if device <> nil then
-        device^.activeRequest := nil;
+        if device^.activeCount > 0 then
+            device^.activeCount := device^.activeCount - 1;
 
     { Invoke completion callback (sync path uses sync_io_callback, async path uses caller's callback) }
     if request^.Callback <> nil then

src/driver/storage/driver.storage.types.pas

@@ -89,7 +89,7 @@ type
         { VFS / FD errors }
         eTooManyOpenFiles,        { per-process FD table full }
         eInvalidHandle,           { TFileHandle does not refer to an open FD }
-        eNotStreamMode,           { SeekFile called on a non-omStream handle }
+
         eFileNotLoaded,           { ReadFile on a pre-load handle whose data is not yet loaded }
         eAlreadyExists,           { generic: target name already exists (symlink, file, etc.) }
 
@@ -140,8 +140,6 @@ type
     TIOCallback = procedure(error : TError; userdata : pointer);
 
     { Filesystem callback core.types }
-    PPWriteHook     = procedure(volume : PStorage_volume; directory : pchar; entry : PDirectory_Entry; byteCount : uint32; buffer : puint32; statusOut : puint32);
-    PPReadHook      = function(volume : PStorage_Volume; directory : pchar; fileName : pchar; buffer : puint32; bytecount : puint32) : uint32;
     PPCreateHook    = procedure(volume : PStorage_volume; start : uint32; size : uint32; config : puint32);
     PPCreateAsyncHook = procedure(volume : PStorage_volume; start : uint32; size : uint32; config : puint32; callback : TIOCallback; callbackData : pointer);
     PPDetectHook    = procedure(disk : PStorage_Device);
@@ -152,23 +150,22 @@ type
     PPIdentifyHook   = function(volume : PStorage_Volume) : boolean;
     PPRenameFileHook = procedure(volume : PStorage_Volume; filePath : pchar; newName : pchar; status : puint32);
     { Offset-based read: reads byteCount bytes starting at byte offset into the file.
-      Returns the number of bytes actually read.
+      Returns the number of bytes actually read. }
-      Filesystems that do not implement this leave the field nil and VFS falls back
-      to the load-all readCallback behaviour. }
     PPReadOffsetHook = function(volume : PStorage_Volume; directory : pchar; fileName : pchar; offset : uint32; buffer : puint32; byteCount : uint32) : uint32;
     { Offset-based write: writes byteCount bytes starting at byte offset into the file.
       Returns the number of bytes actually written.
       Filesystems that do not implement this leave the field nil; VFS returns 0
       for stream-mode writes on volume FDs. }
     PPWriteOffsetHook = function(volume : PStorage_Volume; directory : pchar; fileName : pchar; offset : uint32; buffer : puint32; byteCount : uint32) : uint32;
+    { File size query: returns the size in bytes of the named file.
+      Returns 0 if the file does not exist or has zero length. }
+    PPFileSizeHook = function(volume : PStorage_Volume; directory : pchar; fileName : pchar) : uint32;
 
     { === Async filesystem hook core.types ===
       Each mirrors the corresponding sync hook but receives a TIOCallback + callbackData.
       The implementation must return immediately; the callback is fired when the
       operation completes from worker-process context (IF=1, safe to block). }
     PPLinkedListBase      = ^PLinkedListBase;   { needed for PPReadDirAsyncHook parameter }
-    PPWriteAsyncHook      = procedure(volume : PStorage_Volume; directory : pchar; entry : PDirectory_Entry; byteCount : uint32; buffer : puint32; callback : TIOCallback; callbackData : pointer);
-    PPReadAsyncHook       = procedure(volume : PStorage_Volume; directory : pchar; fileName : pchar; buffer : puint32; byteCount : puint32; callback : TIOCallback; callbackData : pointer);
     PPCreateDirAsyncHook  = procedure(volume : PStorage_Volume; directory : pchar; dirname : pchar; attributes : uint32; status : puint32; callback : TIOCallback; callbackData : pointer);
     PPReadDirAsyncHook    = procedure(volume : PStorage_Volume; directory : pchar; resultList : PPLinkedListBase; status : puint32; callback : TIOCallback; callbackData : pointer);
     PPDeleteFileAsyncHook = procedure(volume : PStorage_Volume; filePath : pchar; status : puint32; callback : TIOCallback; callbackData : pointer);
@@ -211,7 +208,8 @@ type
         dispatchRead     : TDriverDispatch;   { nil if driver not yet migrated }
         dispatchWrite    : TDriverDispatch;   { nil if driver not yet migrated }
         requestQueue     : PCFIFOQueue;       { per-device I/O request CFIFO }
-        activeRequest    : PIORequest;        { currently dispatched to HW, or nil }
+        activeCount      : uint32;            { number of commands currently dispatched to HW }
+        maxActive        : uint32;            { max concurrent commands (NCQ depth) }
 
         maxSectorCount   : uint32;
         sectorSize       : uint32; //in bytes
@@ -224,8 +222,6 @@ type
     TFilesystem = record
         sName              : pchar;
         system_id          : uint8;
-        writeCallback      : PPWriteHook;
-        readCallback       : PPReadHook;
         createCallback     : PPCreateHook;
         detectCallback     : PPDetectHook;
         createDirCallback  : PPCreateDirHook;
@@ -237,12 +233,10 @@ type
         readOffsetCallback : PPReadOffsetHook;
         { Offset-based write for streaming mode — nil if not implemented }
         writeOffsetCallback : PPWriteOffsetHook;
+        { File size query — returns size in bytes without loading the file }
+        fileSizeCallback : PPFileSizeHook;
         { Async format — nil if FS only supports synchronous create }
         createAsyncCallback : PPCreateAsyncHook;
-        { Async variants of the main I/O hooks — nil if FS only supports sync.
-          Sync VFS functions will call these + spin-wait when available. }
-        writeAsyncCallback      : PPWriteAsyncHook;
-        readAsyncCallback       : PPReadAsyncHook;
         createDirAsyncCallback  : PPCreateDirAsyncHook;
         readDirAsyncCallback    : PPReadDirAsyncHook;
         deleteFileAsyncCallback : PPDeleteFileAsyncHook;
@@ -260,6 +254,7 @@ type
         freeSectors  : uint32;
         filesystem   : PFilesystem;
         isBootDrive  : boolean;
+        fsPrivate    : pointer;      { FS-driver private data (e.g. PFATCache for FAT32) }
     end;
 
     { Generic directory entry }

src/driver/storage/fs/driver.storage.fs.flatfs.pas

@@ -843,12 +843,106 @@ begin
     readDirectoryEntries := entries;
 end;
 
+function getFileSize(volume : PStorage_Volume; directory : pchar; fileName : pchar) : uint32;
+var
+    fileEntries : PFile_Entry;
+    fileCount   : uint32;
+    i           : uint32;
+begin
+    getFileSize := 0;
+    fileCount := 1000;
+    fileEntries := PFile_Entry(kalloc(sizeof(TFile_Entry) * fileCount + 512));
+    memset(uint32(fileEntries), 0, sizeof(TFile_Entry) * fileCount);
+    driver.storage.mgr.storage_read(volume^.device, volume^.sectorStart + 1,
+        sizeof(TFile_Entry) * fileCount div 512, puint32(fileEntries));
+
+    for i := 0 to fileCount - 1 do begin
+        if fileEntries[i].attribues = 0 then break;
+        if stringEquals(fileEntries[i].name, fileName) then begin
+            getFileSize := fileEntries[i].size;
+            break;
+        end;
+    end;
+
+    kfree(puint32(fileEntries));
+end;
+
+function readFileAtOffset(volume : PStorage_Volume; directory : pchar;
+                          fileName : pchar; offset : uint32;
+                          buffer : puint32; byteCount : uint32) : uint32;
+var
+    fileEntries  : PFile_Entry;
+    fileCount    : uint32;
+    i            : uint32;
+    fileStart    : uint32;
+    fileSize     : uint32;
+    remaining    : uint32;
+    secOff       : uint32;
+    curSec       : uint32;
+    destPos      : uint32;
+    bytesToCopy  : uint32;
+    secBuf       : puint32;
+    found        : boolean;
+begin
+    readFileAtOffset := 0;
+    fileCount := 1000;
+    found := false;
+
+    fileEntries := PFile_Entry(kalloc(sizeof(TFile_Entry) * fileCount + 512));
+    memset(uint32(fileEntries), 0, sizeof(TFile_Entry) * fileCount);
+    driver.storage.mgr.storage_read(volume^.device, volume^.sectorStart + 1,
+        sizeof(TFile_Entry) * fileCount div 512, puint32(fileEntries));
+
+    for i := 0 to fileCount - 1 do begin
+        if fileEntries[i].attribues = 0 then break;
+        if stringEquals(fileEntries[i].name, fileName) then begin
+            fileStart := fileEntries[i].start;
+            fileSize  := fileEntries[i].size;
+            found := true;
+            break;
+        end;
+    end;
+
+    kfree(puint32(fileEntries));
+    if not found then exit;
+
+    { Clamp to EOF }
+    if offset >= fileSize then exit;
+    remaining := fileSize - offset;
+    if byteCount < remaining then remaining := byteCount;
+
+    secBuf := puint32(kalloc(512));
+    if secBuf = nil then exit;
+
+    destPos := 0;
+    curSec  := fileStart + (offset div 512);
+    secOff  := offset mod 512;
+
+    while remaining > 0 do begin
+        driver.storage.mgr.storage_read(volume^.device, curSec, 1, secBuf);
+
+        bytesToCopy := 512 - secOff;
+        if bytesToCopy > remaining then bytesToCopy := remaining;
+        core.util.memcpy(uint32(secBuf) + secOff, uint32(buffer) + destPos, bytesToCopy);
+
+        destPos   := destPos + bytesToCopy;
+        remaining := remaining - bytesToCopy;
+        secOff    := 0;
+        curSec    := curSec + 1;
+    end;
+
+    kfree(secBuf);
+    readFileAtOffset := destPos;
+end;
+
 procedure init();
 begin
     push_trace('driver.storage.fs.flatfs.init()');
     filesystem.sName:= 'driver.storage.fs.flatfs'; 
     filesystem.system_id:= $02; 
     filesystem.readDirCallback:= @readDirectoryEntries;
+    filesystem.fileSizeCallback:= @getFileSize;
+    filesystem.readOffsetCallback:= @readFileAtOffset;
     // filesystem.createDirCallback:= @writeDirectoryGen;
     filesystem.createcallback:= @create_volume;
     filesystem.detectcallback:= @detect_volumes;

src/driver/storage/vol/driver.storage.vol.mgr.pas

@@ -80,6 +80,9 @@ procedure delete_volume(volume : PStorage_Volume);
 
 implementation
 
+uses
+    driver.storage.vfs;
+
 { ===================== helpers ===================== }
 
 { Create a TStorage_Volume from a partition entry and register it }
@@ -135,8 +138,10 @@ begin
             i := i + 1;
     end;
 
-    if found <> nil then
+    if found <> nil then begin
+        driver.storage.vfs.InvalidateVolume(found);
         kfree(puint32(found));
+    end;
 end;
 
 { Remove all volumes belonging to a device }
@@ -150,6 +155,7 @@ begin
         v := PStorage_Volume(Void(DL_Get(volumes, i))^);
         if v^.device = device then begin
             DL_Delete(volumes, i);
+            driver.storage.vfs.InvalidateVolume(v);
             kfree(puint32(v));
         end else
             i := i + 1;