745 lines
29 KiB
ObjectPascal
745 lines
29 KiB
ObjectPascal
// Copyright 2021 Aaron Hance
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
{
|
|
Driver->Storage->StorageManager - Physical storage device registry + I/O dispatch.
|
|
|
|
Manages registration of physical storage devices (driver.storage.ctl.ahci, driver.storage.ctl.ide, driver.bus.usb, etc.)
|
|
and provides the submit_io / complete_io request-based I/O path.
|
|
Device drivers call register_device() when they discover a new device.
|
|
|
|
Request-based I/O model:
|
|
- submit_io enqueues a TIORequest into the device's per-device CFIFO,
|
|
dispatches to hardware, and returns without waiting for completion.
|
|
- complete_io delivers callbacks from the driver's completion context.
|
|
- storage_read / storage_write are thin wrappers that build a stack-local
|
|
TIORequest, call submit_io_wait, and return the resulting TError.
|
|
- Drivers must provide the TDriverDispatch interface; there is no legacy
|
|
callback fallback.
|
|
|
|
@author(Aaron Hance <[email protected]>)
|
|
}
|
|
unit driver.storage.mgr;
|
|
|
|
interface
|
|
|
|
uses
|
|
arch.x86.bda,
|
|
boot.mgr,
|
|
core.ds.cfifo,
|
|
core.ds.types,
|
|
driver.storage.iorequest,
|
|
core.ds.lists,
|
|
memory.heap,
|
|
driver.storage.vol.mbr,
|
|
proc.mgr,
|
|
proc.types,
|
|
driver.storage.types,
|
|
io.syslog,
|
|
debug.tracer,
|
|
core.util, arch.x86.util;
|
|
|
|
var
|
|
storageDevices : PDList;
|
|
nextDeviceId : uint32;
|
|
|
|
{ BIOS drive byte from arch.x86.multiboot — set once during core.version init before any
|
|
device drivers run. $80=first HDD, $81=second HDD, $9F/other=CD-ROM. }
|
|
bootDriveByte : uint8;
|
|
hdCount : uint32; { non-ATAPI devices registered so far }
|
|
atapiCount : uint32; { ATAPI devices registered so far }
|
|
|
|
procedure storage_get_stats(device : PStorage_Device; var stats : TStorageStats);
|
|
procedure init();
|
|
procedure set_boot_drive_byte(driveByte : uint8);
|
|
function get_boot_drive_byte() : uint8;
|
|
procedure register_device(device : PStorage_Device);
|
|
function get_device_list() : PDList;
|
|
function get_device_count() : uint32;
|
|
function get_device(index : uint32) : PStorage_Device;
|
|
function controller_type_2_string(controllerType : TControllerType) : pchar;
|
|
function read_mbr(device : PStorage_Device) : PMaster_Boot_Record;
|
|
function write_mbr(device : PStorage_Device; mbr : PMaster_Boot_Record) : TError;
|
|
|
|
{ Return pointer to cached mbr data (nil if not cached).
|
|
Caller must NOT free the returned pointer. }
|
|
function get_cached_mbr(device : PStorage_Device) : PMaster_Boot_Record;
|
|
|
|
{ Async mbr write — updates cache and issues non-blocking write.
|
|
Callback fires from ISR context when write completes. }
|
|
procedure write_mbr_async(device : PStorage_Device; mbr : PMaster_Boot_Record;
|
|
callback : TIOCallback; callbackData : pointer);
|
|
|
|
{ === I/O dispatch API === }
|
|
|
|
{ Non-blocking: enqueue request into the device CFIFO and dispatch.
|
|
Returns eNone if the request was successfully enqueued.
|
|
Completion will arrive via the request's Callback (async) or by
|
|
waking request^.Caller (blocking). Caller must set up Callback or
|
|
Caller/UserData BEFORE calling this. }
|
|
function submit_io(request : PIORequest) : TError;
|
|
|
|
{ Blocking wrapper: sets up caller context, calls submit_io, then
|
|
hlt-loops until complete_io wakes us. MUST only be called from
|
|
normal task context (interrupts enabled). }
|
|
function submit_io_wait(request : PIORequest) : TError;
|
|
|
|
{ Called from ISR / completion context to finalise a request.
|
|
Invokes Callback if set (async path) and/or wakes Caller (blocking path). }
|
|
procedure complete_io(request : PIORequest; success : boolean; error : TError);
|
|
|
|
{ Blocking I/O helpers — task context only.
|
|
Build a TIORequest, submit_io_wait, return error. }
|
|
function storage_read(device : PStorage_Device; addr : uint32; sectors : uint32; buffer : puint32) : TError;
|
|
function storage_write(device : PStorage_Device; addr : uint32; sectors : uint32; buffer : puint32) : TError;
|
|
|
|
{ Async I/O helpers — safe from any context (ISR or task).
|
|
Enqueue I/O request and return immediately. Callback fires from
|
|
ISR context when the operation completes. }
|
|
function storage_read_async(device : PStorage_Device; addr : uint32; sectors : uint32;
|
|
buffer : puint32; callback : TIOCallback; callbackData : pointer) : TError;
|
|
function storage_write_async(device : PStorage_Device; addr : uint32; sectors : uint32;
|
|
buffer : puint32; callback : TIOCallback; callbackData : pointer) : TError;
|
|
|
|
implementation
|
|
|
|
uses
|
|
driver.storage.vol.mgr;
|
|
|
|
{ Internal: dequeue head of device queue and dispatch to driver hardware }
|
|
procedure dispatch_next(device : PStorage_Device); forward;
|
|
|
|
type
|
|
{ Used by submit_io_wait: callback writes result here then sets Done=1.
|
|
Process is parked in psAwaiting while Done=0; ISR sets psReady + Done=1.
|
|
Done is read through a pointer so FPC cannot cache it (Lesson #6). }
|
|
TSyncWaitData = record
|
|
Done : uint32; { 0 = in flight, 1 = complete }
|
|
Error : TError;
|
|
Process : PProcessContext; { process to wake from ISR context }
|
|
end;
|
|
PSyncWaitData = ^TSyncWaitData;
|
|
|
|
{ Async MBR-write wrapper. Keeps a sector-sized DMA buffer alive until the
|
|
hardware completion fires, then frees it before forwarding the user's callback. }
|
|
TMBRWriteAsyncData = record
|
|
Buffer : pointer;
|
|
Device : PStorage_Device;
|
|
UserCallback : TIOCallback;
|
|
UserData : pointer;
|
|
end;
|
|
PMBRWriteAsyncData = ^TMBRWriteAsyncData;
|
|
|
|
{ Callback fired from ISR context by complete_io.
|
|
Sets the error, wakes the parked process, then sets Done=1 so the
|
|
pointer-dereferenced spin in submit_io_wait can exit. Safe with IF=0. }
|
|
procedure sync_io_callback(error : TError; userdata : pointer);
|
|
var
|
|
wait : PSyncWaitData;
|
|
begin
|
|
if userdata = nil then exit;
|
|
wait := PSyncWaitData(userdata);
|
|
wait^.Error := error;
|
|
if wait^.Process <> nil then
|
|
wait^.Process^.State := psReady;
|
|
{ Write Done last — submit_io_wait spins on this and must see State=psReady
|
|
before it exits the loop (ISR ordering on x86 is program order). }
|
|
puint32(@wait^.Done)^ := 1;
|
|
end;
|
|
|
|
function alloc_mbr_io_buffer(device : PStorage_Device; mbr : PMaster_Boot_Record) : puint32;
|
|
var
|
|
bufSz : uint32;
|
|
buf : puint32;
|
|
begin
|
|
alloc_mbr_io_buffer := nil;
|
|
if (device = nil) or (mbr = nil) then exit;
|
|
|
|
bufSz := device^.sectorSize;
|
|
if bufSz < sizeof(TMaster_Boot_Record) then
|
|
bufSz := sizeof(TMaster_Boot_Record);
|
|
|
|
buf := puint32(kalloc(bufSz));
|
|
if buf = nil then exit;
|
|
|
|
memset(uint32(buf), 0, bufSz);
|
|
memcpy(uint32(mbr), uint32(buf), sizeof(TMaster_Boot_Record));
|
|
alloc_mbr_io_buffer := buf;
|
|
end;
|
|
|
|
function claim_mbr(device : PStorage_Device) : boolean;
|
|
var flags : uint32;
|
|
begin
|
|
asm pushfd; cli; pop flags end;
|
|
claim_mbr := not device^.mbrWriteBusy;
|
|
if claim_mbr then device^.mbrWriteBusy := true;
|
|
if (flags and $200) <> 0 then asm sti end;
|
|
end;
|
|
|
|
procedure publish_mbr(device : PStorage_Device; buffer : pointer);
|
|
var old : pointer;
|
|
begin
|
|
{ The DMA buffer becomes the cache only after a successful write. }
|
|
old := device^.cachedMBR;
|
|
device^.cachedMBR := buffer;
|
|
if old <> nil then kfree(old);
|
|
end;
|
|
|
|
procedure mbr_write_async_callback(error : TError; userdata : pointer);
|
|
var ctx : PMBRWriteAsyncData;
|
|
begin
|
|
ctx := PMBRWriteAsyncData(userdata);
|
|
if ctx = nil then exit;
|
|
if error = eNone then publish_mbr(ctx^.Device, ctx^.Buffer)
|
|
else kfree(ctx^.Buffer);
|
|
ctx^.Device^.mbrWriteBusy := false;
|
|
if ctx^.UserCallback <> nil then ctx^.UserCallback(error, ctx^.UserData);
|
|
kfree(void(ctx));
|
|
end;
|
|
|
|
procedure storage_get_stats(device : PStorage_Device; var stats : TStorageStats);
|
|
begin
|
|
asm pushf; cli end;
|
|
if device <> nil then stats := device^.Stats
|
|
else memset(uint32(@stats),0,sizeof(stats));
|
|
asm popf end;
|
|
end;
|
|
|
|
procedure init();
|
|
begin
|
|
push_trace('driver.storage.mgr.init');
|
|
storageDevices := DL_New(sizeof(TStorage_Device));
|
|
nextDeviceId := 0;
|
|
bootDriveByte := $FF; { $FF = unknown/uninitialised }
|
|
hdCount := 0;
|
|
atapiCount := 0;
|
|
end;
|
|
|
|
procedure set_boot_drive_byte(driveByte : uint8);
|
|
begin
|
|
bootDriveByte := driveByte;
|
|
io.syslog.log('STRMGR', 'Boot drive BIOS byte = 0x');
|
|
io.syslog.writehexln(driveByte);
|
|
end;
|
|
|
|
function get_boot_drive_byte() : uint8;
|
|
begin
|
|
get_boot_drive_byte := bootDriveByte;
|
|
end;
|
|
|
|
procedure register_device(device : PStorage_Device);
|
|
var
|
|
storedDev : PStorage_Device;
|
|
begin
|
|
push_trace('driver.storage.mgr.register_device');
|
|
device^.id := nextDeviceId;
|
|
nextDeviceId := nextDeviceId + 1;
|
|
|
|
{ Initialise new Phase 2 fields }
|
|
device^.removed := false;
|
|
device^.requestQueue := CFIFO_New(SizeOf(PIORequest), 128);
|
|
memset(uint32(@device^.Stats),0,sizeof(TStorageStats));
|
|
device^.activeCount := 0;
|
|
device^.dispatching := false;
|
|
if device^.maxActive = 0 then device^.maxActive := 1;
|
|
device^.cachedMBR := nil;
|
|
device^.metadataBusy := false;
|
|
device^.mbrWriteBusy := false;
|
|
device^.isBootDevice := false;
|
|
{ dispatchRead/dispatchWrite: left as-is — driver sets them before
|
|
calling register_device. memset(0) guarantees nil for unmigrated drivers. }
|
|
|
|
DL_Add(storageDevices);
|
|
DL_Set(storageDevices, DL_Size(storageDevices) - 1, puint32(device));
|
|
|
|
{ Get pointer to the DList copy so volumes reference the canonical device }
|
|
storedDev := PStorage_Device(DL_Get(storageDevices, DL_Size(storageDevices) - 1));
|
|
|
|
{ Identify whether this is the boot device using the BIOS drive byte.
|
|
BIOS numbers HDDs from $80 upward; the nth HDD is $80 + (n-1).
|
|
ATAPI/CD-ROM devices are NOT in the $80-range HDD sequence, so if
|
|
bootDriveByte does not match any HDD we have seen, the first ATAPI
|
|
device is marked as the boot device (covers standard ISO boot). }
|
|
if (storedDev^.controller = ControllerATAPI) or
|
|
(storedDev^.controller = ControllerAHCI_ATAPI) then begin
|
|
{ ATAPI device: boot if the drive byte is outside the known HDD range }
|
|
if bootDriveByte < ($80 + hdCount) then begin
|
|
{ drive byte refers to an HDD — this ATAPI is not the boot device }
|
|
end else if atapiCount = 0 then begin
|
|
{ first ATAPI and boot byte is not a known HDD → this is the boot drive }
|
|
storedDev^.isBootDevice := true;
|
|
io.syslog.logln('STRMGR', 'Boot device identified (ATAPI/ISO).');
|
|
end;
|
|
atapiCount := atapiCount + 1;
|
|
end else begin
|
|
{ Non-ATAPI (HDD): boot if BIOS drive byte matches $80 + index }
|
|
if bootDriveByte = ($80 + hdCount) then begin
|
|
storedDev^.isBootDevice := true;
|
|
io.syslog.logln('STRMGR', 'Boot device identified (HDD).');
|
|
end;
|
|
hdCount := hdCount + 1;
|
|
end;
|
|
|
|
{ Discover partitions and register volumes automatically }
|
|
if storedDev^.maxSectorCount > 0 then
|
|
driver.storage.vol.mgr.discover_volumes(storedDev);
|
|
end;
|
|
|
|
function get_device_list() : PDList;
|
|
begin
|
|
get_device_list := storageDevices;
|
|
end;
|
|
|
|
function get_device_count() : uint32;
|
|
begin
|
|
get_device_count := DL_Size(storageDevices);
|
|
end;
|
|
|
|
function get_device(index : uint32) : PStorage_Device;
|
|
begin
|
|
if index < DL_Size(storageDevices) then
|
|
get_device := PStorage_Device(DL_Get(storageDevices, index))
|
|
else
|
|
get_device := nil;
|
|
end;
|
|
|
|
function controller_type_2_string(controllerType : TControllerType) : pchar;
|
|
begin
|
|
case controllerType of
|
|
ControllerATA: controller_type_2_string := 'ATA';
|
|
ControllerATAPI: controller_type_2_string := 'ATAPI';
|
|
ControllerUSB: controller_type_2_string := 'USB';
|
|
ControllerAHCI: controller_type_2_string := 'AHCI';
|
|
ControllerAHCI_ATAPI: controller_type_2_string := 'AHCI_ATAPI';
|
|
ControllerNVMe: controller_type_2_string := 'NVMe';
|
|
ControllerNET: controller_type_2_string := 'NET';
|
|
ControllerRAM: controller_type_2_string := 'RAM';
|
|
ControllerSCSI: controller_type_2_string := 'SCSI';
|
|
end;
|
|
end;
|
|
|
|
function read_mbr(device : PStorage_Device) : PMaster_Boot_Record;
|
|
var
|
|
err : TError;
|
|
bufSz : uint32;
|
|
buf : puint32;
|
|
cache : puint32;
|
|
begin
|
|
push_trace('driver.storage.mgr.read_mbr');
|
|
{ Allocate at least one full sector so DMA does not overrun
|
|
the buffer on devices with large sectors (e.g. 2048-byte ATAPI). }
|
|
bufSz := device^.sectorSize;
|
|
if bufSz < sizeof(TMaster_Boot_Record) then
|
|
bufSz := sizeof(TMaster_Boot_Record);
|
|
buf := puint32(kalloc(bufSz));
|
|
read_mbr := PMaster_Boot_Record(buf);
|
|
err := storage_read(device, 0, 1, buf);
|
|
if err <> eNone then begin
|
|
kfree(void(buf));
|
|
read_mbr := nil;
|
|
end else begin
|
|
{ Update cache with a copy }
|
|
if device^.cachedMBR <> nil then
|
|
kfree(device^.cachedMBR);
|
|
cache := puint32(kalloc(sizeof(TMaster_Boot_Record)));
|
|
if cache <> nil then
|
|
memcpy(uint32(buf), uint32(cache), sizeof(TMaster_Boot_Record));
|
|
device^.cachedMBR := pointer(cache);
|
|
end;
|
|
end;
|
|
|
|
function write_mbr(device : PStorage_Device; mbr : PMaster_Boot_Record) : TError;
|
|
var buf : puint32;
|
|
begin
|
|
write_mbr := eInvalidArgument;
|
|
if (device = nil) or (mbr = nil) then exit;
|
|
if not device^.writable then begin write_mbr := eReadOnly; exit; end;
|
|
if not claim_mbr(device) then begin write_mbr := eFileInUse; exit; end;
|
|
{ Snapshot first: mbr may alias device^.cachedMBR. }
|
|
buf := alloc_mbr_io_buffer(device, mbr);
|
|
if buf = nil then write_mbr := eOutOfMemory
|
|
else begin
|
|
write_mbr := storage_write(device, 0, 1, buf);
|
|
if write_mbr = eNone then publish_mbr(device, buf) else kfree(buf);
|
|
end;
|
|
device^.mbrWriteBusy := false;
|
|
end;
|
|
|
|
function get_cached_mbr(device : PStorage_Device) : PMaster_Boot_Record;
|
|
begin
|
|
if device = nil then begin get_cached_mbr := nil; exit; end;
|
|
get_cached_mbr := PMaster_Boot_Record(device^.cachedMBR);
|
|
end;
|
|
|
|
procedure write_mbr_async(device : PStorage_Device; mbr : PMaster_Boot_Record;
|
|
callback : TIOCallback; callbackData : pointer);
|
|
var ctx : PMBRWriteAsyncData; buf : puint32; err : TError;
|
|
begin
|
|
err := eNone;
|
|
if (device = nil) or (mbr = nil) then err := eInvalidArgument
|
|
else if not device^.writable then err := eReadOnly
|
|
else if not claim_mbr(device) then err := eFileInUse;
|
|
if err <> eNone then begin
|
|
if callback <> nil then callback(err, callbackData);
|
|
exit;
|
|
end;
|
|
buf := alloc_mbr_io_buffer(device, mbr);
|
|
if buf = nil then begin
|
|
device^.mbrWriteBusy := false;
|
|
if callback <> nil then callback(eOutOfMemory, callbackData);
|
|
exit;
|
|
end;
|
|
ctx := PMBRWriteAsyncData(kalloc(sizeof(TMBRWriteAsyncData)));
|
|
if ctx = nil then begin
|
|
kfree(buf); device^.mbrWriteBusy := false;
|
|
if callback <> nil then callback(eOutOfMemory, callbackData);
|
|
exit;
|
|
end;
|
|
ctx^.Buffer := buf; ctx^.Device := device;
|
|
ctx^.UserCallback := callback; ctx^.UserData := callbackData;
|
|
err := storage_write_async(device, 0, 1, buf, @mbr_write_async_callback, ctx);
|
|
if err <> eNone then mbr_write_async_callback(err, ctx);
|
|
end;
|
|
|
|
{ ========================================================================== }
|
|
{ Phase 2: Request-Based I/O Path }
|
|
{ ========================================================================== }
|
|
|
|
{ Internal: dequeue head of device queue and dispatch to hardware.
|
|
Called with interrupts disabled (CLI context) or from ISR context. }
|
|
procedure dispatch_next(device : PStorage_Device);
|
|
var
|
|
heapReq, nextReq, tail : PIORequest;
|
|
peek : pointer;
|
|
merged, bytes, ticks : uint32;
|
|
begin
|
|
if device^.dispatching then exit;
|
|
device^.dispatching := true;
|
|
{ Dispatch queued requests up to maxActive hardware slots }
|
|
while device^.activeCount < device^.maxActive do begin
|
|
if not CFIFO_Dequeue(device^.requestQueue, @heapReq) then break; { queue empty }
|
|
if heapReq = nil then
|
|
break;
|
|
|
|
{ Merge only consecutive FIFO entries with consecutive DMA buffers.
|
|
Bound both command size and completion fan-out; never reorder writes. }
|
|
tail := heapReq; merged := 1;
|
|
while (merged < 32) and (device^.mergeMaxSectors > 0) and
|
|
(device^.sectorSize > 0) do begin
|
|
peek := CFIFO_Peek(device^.requestQueue);
|
|
if peek = nil then break;
|
|
nextReq := PIORequest(puint32(peek)^);
|
|
if nextReq = nil then break;
|
|
if (nextReq^.Device <> device) or (nextReq^.RequestType <> heapReq^.RequestType) then break;
|
|
if heapReq^.SectorCount > device^.mergeMaxSectors then break;
|
|
if nextReq^.SectorCount > device^.mergeMaxSectors-heapReq^.SectorCount then break;
|
|
if heapReq^.SectorCount > ($FFFFFFFF div device^.sectorSize) then break;
|
|
bytes := heapReq^.SectorCount*device^.sectorSize;
|
|
if (heapReq^.LBA > $FFFFFFFF-heapReq^.SectorCount) or
|
|
(uint32(heapReq^.Buffer) > $FFFFFFFF-bytes) then break;
|
|
if (nextReq^.LBA <> heapReq^.LBA+heapReq^.SectorCount) or
|
|
(uint32(nextReq^.Buffer) <> uint32(heapReq^.Buffer)+bytes) then break;
|
|
if not CFIFO_Dequeue(device^.requestQueue,@nextReq) then break;
|
|
tail^.MergeNext := nextReq; tail := nextReq;
|
|
Inc(heapReq^.SectorCount,nextReq^.SectorCount); Inc(merged);
|
|
Inc(device^.Stats.Merged);
|
|
end;
|
|
ticks := puint32(@Counters.c32)^;
|
|
tail := heapReq;
|
|
while tail <> nil do begin
|
|
tail^.DispatchTick := ticks; tail^.State := iosDispatched;
|
|
bytes := ticks-tail^.QueuedTick;
|
|
Inc(device^.Stats.QueueTicks,bytes);
|
|
if bytes > device^.Stats.MaxQueueTicks then device^.Stats.MaxQueueTicks := bytes;
|
|
tail := tail^.MergeNext;
|
|
end;
|
|
Inc(device^.Stats.Commands);
|
|
device^.activeCount := device^.activeCount + 1;
|
|
if device^.activeCount > device^.Stats.ActivePeak then device^.Stats.ActivePeak := device^.activeCount;
|
|
|
|
case heapReq^.RequestType of
|
|
ioRead: begin
|
|
if device^.dispatchRead <> nil then
|
|
device^.dispatchRead(device, heapReq)
|
|
else begin
|
|
complete_io(heapReq, false, eDeviceNotFound);
|
|
end;
|
|
end;
|
|
ioWrite: begin
|
|
if device^.dispatchWrite <> nil then
|
|
device^.dispatchWrite(device, heapReq)
|
|
else
|
|
complete_io(heapReq, false, eDeviceNotFound);
|
|
end;
|
|
end;
|
|
end;
|
|
device^.dispatching := false;
|
|
end;
|
|
|
|
function submit_io(request : PIORequest) : TError;
|
|
var
|
|
device : PStorage_Device;
|
|
heapReq : PIORequest;
|
|
flags : uint32;
|
|
begin
|
|
{ Pre-checks — fail fast before touching the queue }
|
|
if request = nil then begin submit_io := eInvalidArgument; exit; end;
|
|
if request^.Device = nil then begin submit_io := eDeviceNotFound; exit; end;
|
|
if request^.Buffer = nil then begin submit_io := eInvalidArgument; exit; end;
|
|
if request^.SectorCount = 0 then begin submit_io := eInvalidArgument; exit; end;
|
|
|
|
device := request^.Device;
|
|
if device^.removed then begin submit_io := eDeviceRemoved; exit; end;
|
|
|
|
{ Device must have dispatch interface }
|
|
if (device^.dispatchRead = nil) and (device^.dispatchWrite = nil) then begin
|
|
submit_io := eDeviceNotFound;
|
|
exit;
|
|
end;
|
|
|
|
request^.State := iosPending;
|
|
request^.Error := eNone;
|
|
|
|
{ Copy once into the pooled request arena so the driver/ISR path never
|
|
depends on a caller-owned stack record. The queue stores pointers to
|
|
these owned requests; complete_io returns them to the pool. }
|
|
heapReq := ioreq_copy(request);
|
|
if heapReq = nil then begin
|
|
submit_io := eOutOfMemory;
|
|
exit;
|
|
end;
|
|
|
|
{ Enqueue and attempt dispatch — must be ISR-safe and bounded.
|
|
Reject once the fixed queue is full instead of growing with CLI held. }
|
|
asm pushfd; cli; pop flags end;
|
|
if CFIFO_Size(device^.requestQueue) >= CFIFO_Capacity(device^.requestQueue) then begin
|
|
Inc(device^.Stats.QueueFull);
|
|
if (flags and $200) <> 0 then asm sti end;
|
|
ioreq_free(heapReq);
|
|
submit_io := eQueueFull;
|
|
exit;
|
|
end;
|
|
heapReq^.QueuedTick := puint32(@Counters.c32)^;
|
|
Inc(device^.Stats.Submitted);
|
|
if request^.SectorCount <= 8 then Inc(device^.Stats.SmallRequests)
|
|
else if request^.SectorCount <= 256 then Inc(device^.Stats.MediumRequests)
|
|
else Inc(device^.Stats.LargeRequests);
|
|
CFIFO_Enqueue(device^.requestQueue, @heapReq);
|
|
if CFIFO_Size(device^.requestQueue) > device^.Stats.QueuePeak then
|
|
device^.Stats.QueuePeak := CFIFO_Size(device^.requestQueue);
|
|
dispatch_next(device);
|
|
if (flags and $200) <> 0 then asm sti end;
|
|
|
|
submit_io := eNone;
|
|
end;
|
|
|
|
{ Blocking wrapper — submit the request then park the calling process until
|
|
the driver.storage.ctl.ahci ISR fires the completion callback.
|
|
|
|
The process is placed in psAwaiting so the scheduler skips it entirely.
|
|
sync_io_callback sets State := psReady then Done := 1. We spin on Done
|
|
through a pointer (Lesson #6 — prevents FPC from caching the value in a
|
|
register across the hlt instruction). }
|
|
function submit_io_wait(request : PIORequest) : TError;
|
|
var
|
|
wait : TSyncWaitData;
|
|
err : TError;
|
|
begin
|
|
wait.Done := 0;
|
|
wait.Error := eNone;
|
|
wait.Process := CurrentProcess;
|
|
|
|
request^.Callback := @sync_io_callback;
|
|
request^.CallbackData := @wait;
|
|
request^.Caller := nil;
|
|
request^.UserData := nil;
|
|
|
|
err := submit_io(request);
|
|
if err <> eNone then begin
|
|
submit_io_wait := err;
|
|
exit;
|
|
end;
|
|
|
|
{ Race-free park: disable interrupts while checking Done and setting
|
|
psAwaiting so the callback cannot fire in the narrow window between
|
|
the Done=0 check and the state write — which would otherwise lose
|
|
the wake signal and leave the process parked forever. }
|
|
asm pushf; cli end;
|
|
if puint32(@wait.Done)^ = 0 then begin
|
|
if CurrentProcess <> nil then
|
|
CurrentProcess^.State := psAwaiting;
|
|
end;
|
|
asm popf end;
|
|
|
|
{ Spin on Done via pointer — mandatory per Lesson #6. FPC would cache
|
|
the value in a register without the indirection, causing an infinite
|
|
loop even after the ISR sets Done=1. HLT yields the CPU each iteration
|
|
so we do not burn cycles; the next interrupt (driver.storage.ctl.ahci or timer) wakes us. }
|
|
while puint32(@wait.Done)^ = 0 do
|
|
asm hlt end;
|
|
|
|
submit_io_wait := wait.Error;
|
|
end;
|
|
|
|
procedure complete_io(request : PIORequest; success : boolean; error : TError);
|
|
var device : PStorage_Device; next : PIORequest; ticks, flags : uint32;
|
|
wasDispatching : boolean;
|
|
begin
|
|
if request = nil then exit;
|
|
device := request^.Device;
|
|
asm pushfd; cli; pop flags end;
|
|
wasDispatching := false;
|
|
if device <> nil then begin
|
|
wasDispatching := device^.dispatching;
|
|
device^.dispatching := true; { callbacks cannot dispatch inside merged fan-out }
|
|
if device^.activeCount > 0 then Dec(device^.activeCount);
|
|
ticks := puint32(@Counters.c32)^-request^.DispatchTick;
|
|
Inc(device^.Stats.ServiceTicks,ticks);
|
|
if ticks > device^.Stats.MaxServiceTicks then device^.Stats.MaxServiceTicks := ticks;
|
|
end;
|
|
while request <> nil do begin
|
|
next := request^.MergeNext;
|
|
if success then begin request^.State := iosComplete; request^.Error := eNone; end
|
|
else begin request^.State := iosError; request^.Error := error; end;
|
|
if device <> nil then begin
|
|
Inc(device^.Stats.Completed);
|
|
if not success then Inc(device^.Stats.Failed)
|
|
else if request^.RequestType = ioRead then Inc(device^.Stats.ReadSectors,request^.OriginalSectors)
|
|
else Inc(device^.Stats.WriteSectors,request^.OriginalSectors);
|
|
end;
|
|
if request^.Callback <> nil then request^.Callback(request^.Error,request^.CallbackData);
|
|
ioreq_free(request); request := next;
|
|
end;
|
|
if device <> nil then begin
|
|
device^.dispatching := wasDispatching;
|
|
dispatch_next(device);
|
|
end;
|
|
if (flags and $200) <> 0 then asm sti end;
|
|
end;
|
|
|
|
{ ========================================================================== }
|
|
{ Public storage_read / storage_write }
|
|
{ ========================================================================== }
|
|
|
|
{ Blocking — task context only }
|
|
function storage_read(device : PStorage_Device; addr : uint32; sectors : uint32; buffer : puint32) : TError;
|
|
var
|
|
req : TIORequest;
|
|
begin
|
|
if device = nil then begin storage_read := eDeviceNotFound; exit; end;
|
|
if buffer = nil then begin storage_read := eInvalidArgument; exit; end;
|
|
if sectors = 0 then begin storage_read := eInvalidArgument; exit; end;
|
|
|
|
req.RequestType := ioRead;
|
|
req.State := iosPending;
|
|
req.Device := device;
|
|
req.LBA := addr;
|
|
req.SectorCount := sectors;
|
|
req.Buffer := pointer(buffer);
|
|
req.ByteCount := 0;
|
|
req.Error := eNone;
|
|
req.Caller := nil;
|
|
req.UserData := nil;
|
|
req.Callback := nil;
|
|
req.CallbackData := nil;
|
|
|
|
storage_read := submit_io_wait(@req);
|
|
end;
|
|
|
|
{ Blocking — task context only }
|
|
function storage_write(device : PStorage_Device; addr : uint32; sectors : uint32; buffer : puint32) : TError;
|
|
var
|
|
req : TIORequest;
|
|
begin
|
|
if device = nil then begin storage_write := eDeviceNotFound; exit; end;
|
|
if buffer = nil then begin storage_write := eInvalidArgument; exit; end;
|
|
if sectors = 0 then begin storage_write := eInvalidArgument; exit; end;
|
|
|
|
req.RequestType := ioWrite;
|
|
req.State := iosPending;
|
|
req.Device := device;
|
|
req.LBA := addr;
|
|
req.SectorCount := sectors;
|
|
req.Buffer := pointer(buffer);
|
|
req.ByteCount := 0;
|
|
req.Error := eNone;
|
|
req.Caller := nil;
|
|
req.UserData := nil;
|
|
req.Callback := nil;
|
|
req.CallbackData := nil;
|
|
|
|
storage_write := submit_io_wait(@req);
|
|
end;
|
|
|
|
{ ========================================================================== }
|
|
{ Async storage_read / storage_write }
|
|
{ ========================================================================== }
|
|
|
|
{ Non-blocking — safe from ISR or task context. Callback fires from ISR. }
|
|
function storage_read_async(device : PStorage_Device; addr : uint32; sectors : uint32;
|
|
buffer : puint32; callback : TIOCallback; callbackData : pointer) : TError;
|
|
var
|
|
req : TIORequest;
|
|
begin
|
|
if device = nil then begin storage_read_async := eDeviceNotFound; exit; end;
|
|
if buffer = nil then begin storage_read_async := eInvalidArgument; exit; end;
|
|
if sectors = 0 then begin storage_read_async := eInvalidArgument; exit; end;
|
|
|
|
req.RequestType := ioRead;
|
|
req.State := iosPending;
|
|
req.Device := device;
|
|
req.LBA := addr;
|
|
req.SectorCount := sectors;
|
|
req.Buffer := pointer(buffer);
|
|
req.ByteCount := 0;
|
|
req.Error := eNone;
|
|
req.Caller := nil;
|
|
req.UserData := nil;
|
|
req.Callback := callback;
|
|
req.CallbackData := callbackData;
|
|
|
|
storage_read_async := submit_io(@req);
|
|
end;
|
|
|
|
{ Non-blocking — safe from ISR or task context. Callback fires from ISR. }
|
|
function storage_write_async(device : PStorage_Device; addr : uint32; sectors : uint32;
|
|
buffer : puint32; callback : TIOCallback; callbackData : pointer) : TError;
|
|
var
|
|
req : TIORequest;
|
|
begin
|
|
if device = nil then begin storage_write_async := eDeviceNotFound; exit; end;
|
|
if buffer = nil then begin storage_write_async := eInvalidArgument; exit; end;
|
|
if sectors = 0 then begin storage_write_async := eInvalidArgument; exit; end;
|
|
|
|
req.RequestType := ioWrite;
|
|
req.State := iosPending;
|
|
req.Device := device;
|
|
req.LBA := addr;
|
|
req.SectorCount := sectors;
|
|
req.Buffer := pointer(buffer);
|
|
req.ByteCount := 0;
|
|
req.Error := eNone;
|
|
req.Caller := nil;
|
|
req.UserData := nil;
|
|
req.Callback := callback;
|
|
req.CallbackData := callbackData;
|
|
|
|
storage_write_async := submit_io(@req);
|
|
end;
|
|
|
|
initialization
|
|
boot.mgr.registerBoot('driver.storage.mgr', @init, 'Storage Manager', 'driver.storage.vfs');
|
|
|
|
end.
|