//  Copyright 2021 Aaron Hance
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

{ 
	Drivers->Storage->driver.storage.ctl.ahci->driver.storage.ctl.ahci - driver.storage.ctl.ahci Driver.
	
	@author(Aaron Hance <ah@aaronhance.me>)
}
unit driver.storage.ctl.ahci;

interface

uses
    boot.mgr,
    driver.storage.ctl.ahci.types,
    driver.mgr,
    driver.types,
    driver.storage.ctl.ide.types,
    arch.x86.isr.ioapic,
    arch.x86.isr.mgr,
    core.ds.lists,
    memory.heap,
    driver.bus.pci,
    driver.storage.mgr,
    driver.storage.types,
    io.syslog,
    core.util, arch.x86.util,
    arch.x86.memory.virtual;

var
    ahciControllers : PDList;
    page_base : puint32;

procedure UnitTest;
procedure init();
function load(ptr : void) : boolean;
procedure check_ports(controller : PAHCI_Controller);
function identify_device(controller : PAHCI_Controller; portIndex : uint32; isATAPI : boolean) : boolean;
procedure ahci_isr();
function find_cmd_slot(device : PAHCI_Device) : uint32;

{ Async primitives — return immediately after issuing the command.
  completion is called from IRQ context when the slot clears.
  completion may be nil if no callback is needed (e.g. init-time polling). }
function send_read_dma_async(device : PAHCI_Device; lba : uint64; count : uint32; buffer : puint32; completion : TIOCompletion; userdata : puint32) : TError;
function send_write_dma_async(device : PAHCI_Device; lba : uint64; count : uint32; buffer : puint32; completion : TIOCompletion; userdata : puint32) : TError;
function read_atapi_async(device : PAHCI_Device; lba : uint64; count : uint32; buffer : puint32; completion : TIOCompletion; userdata : puint32) : TError;

{ Storage-manager async callback wrappers }
{ Phase 3: TDriverDispatch procedures — bridge PIORequest to driver.storage.ctl.ahci DMA }
procedure ahci_dispatch_read(device : PStorage_Device; request : PIORequest);
procedure ahci_dispatch_write(device : PStorage_Device; request : PIORequest);
procedure ahci_dispatch_atapi_read(device : PStorage_Device; request : PIORequest);

function send_read_capacity(device : PAHCI_Device; sectorCount : puint32; blockSize : puint32) : boolean;


implementation

uses arch.x86.bda, arch.x86.isr.tmr0, core.panic, io.stdio, core.strings;

const
    AHCI_ENABLE_NCQ = true;
    AHCI_IO_TICKS = 80000; { about 10 seconds at the ~8 kHz PIT }
    AHCI_STOP_TICKS = 8000;
    AHCI_FATAL_MASK = $79000000; { TFES, HBFS, HBDS, IFS, OFS }
    AHCI_IRQ_MASK = $7900000B;

var watchdogDivider : uint32 = 0;

procedure cmd_ahcitest(params : PParamList; stdin_buf, stdout_buf, stderr_buf : POutBuf); forward;
procedure ahci_watchdog(data : void); forward;
procedure fail_controller(controller : PAHCI_Controller; error : TError); forward;

function min_u32(a : uint32; b : uint32) : uint32;
begin
    if a < b then
        min_u32 := a
    else
        min_u32 := b;
end;

procedure init();
var
    devID : TDeviceIdentifier;
begin
    io.syslog.writestringln('driver.storage.ctl.ahci: Registering driver');
    devID.bus:= biPCI;
    devID.id0:= idANY;
    devID.id1:= $00000001;
    devID.id2:= $00000006;
    devID.id3:= $00000001;
    devID.id4:= idANY;
    devID.ex:= nil;
    driver.mgr.register_driver('ATA/PI driver.storage.ctl.ahci Driver', @devID, @load);
    arch.x86.isr.tmr0.hook(uint32(@ahci_watchdog));
    io.stdio.registerCommand('AHCITEST',@cmd_ahcitest,'Run AHCI tests using simulated registers');
end;

{ Poll MMIO through a pointer on every iteration. The PIT runs at ~8 kHz;
  16 psleep ticks give a conservative ~2 ms poll interval, not a CPU-speed loop. }
function wait_register(reg : puint32; mask, expected, attempts : uint32) : boolean;
var n : uint32;
begin
    n := 0;
    repeat
        if (reg^ and mask) = expected then begin wait_register := true; exit; end;
        if n >= attempts then break;
        psleep(16); Inc(n);
    until false;
    wait_register := false;
end;

function stop_port(port : PHBA_Port) : boolean;
begin
    port^.cmd := port^.cmd and not $1;
    stop_port := wait_register(@port^.cmd,$8000,0,250);
    if not stop_port then exit;
    port^.cmd := port^.cmd and not $10;
    stop_port := wait_register(@port^.cmd,$4000,0,250);
end;

function start_port(port : PHBA_Port) : boolean;
begin
    start_port := wait_register(@port^.cmd,$8000,0,250);
    if not start_port then exit;
    port^.cmd := port^.cmd or $10;
    port^.cmd := port^.cmd or $1;
end;

{ DMA addresses must already be installed before enabling FIS reception. }
function reset_port(port : PHBA_Port) : boolean;
begin
    reset_port := false;
    port^.cmd := port^.cmd and not $1;
    if not wait_register(@port^.cmd,$8000,0,250) then exit;
    port^.int_enable := 0;
    port^.cmd := port^.cmd or $16; { FRE, power-on and spin-up }
    port^.sata_ctrl := (port^.sata_ctrl and $FFFFFFF0) or $1;
    psleep(16); { >= 1 ms COMRESET assertion at the ~8 kHz PIT rate }
    port^.sata_ctrl := port^.sata_ctrl and $FFFFFFF0;
    if not wait_register(@port^.sata_status,$F,3,500) then exit;
    if not wait_register(@port^.tfd,$88,0,2500) then exit;
    port^.sata_error := $FFFFFFFF;
    port^.int_status := $FFFFFFFF;
    reset_port := start_port(port);
end;

procedure log_port_failure(portIndex : uint32; port : PHBA_Port; reason : pchar);
begin
    io.syslog.log('AHCI','Port '); io.syslog.writeint(portIndex);
    io.syslog.writestring(': '); io.syslog.writestring(reason);
    io.syslog.writestring(' SSTS='); io.syslog.writehex(port^.sata_status);
    io.syslog.writestring(' SIG='); io.syslog.writehex(port^.signature);
    io.syslog.writestring(' TFD='); io.syslog.writehexln(port^.tfd);
end;

{
    Check the ports on the controller and setup the command list, FIS, and command table
}
procedure check_ports(controller : PAHCI_Controller);
var
    i : uint32;
    ii, attempt, signatureWait : uint32;
    registered : boolean;
    listAlloc, fisAlloc, tableAlloc : puint32;
    port : PHBA_Port;
    device : PAHCI_Device;
    cmd_list_base : puint32;
    fis_base : puint32;
    cmd_table_base : puint32;
    command_header : PHBA_CMD_HEADER;
begin
    for i:=0 to 31 do begin
        if controller^.offline then break;
        if ((controller^.mio^.ports_implimented shr i) and 1) = 1 then begin
            port := @controller^.mio^.ports[i];

            device := PAHCI_Device(@controller^.devices[i]);
            device^.port := port; device^.owner := controller;

            port^.int_enable := 0;
            if not stop_port(port) then begin log_port_failure(i,port,'cannot stop engine'); continue; end;
            { Bring up power before checking for an attached device. }
            port^.cmd := port^.cmd or $6;
            if not wait_register(@port^.sata_status,$F,3,100) then begin
                if (port^.sata_status and $F) = 0 then continue;
                { Present but not yet communicating: COMRESET below may recover it. }
            end;
            //allocate memory for the command list and ensure it is aligned to 1024 bytes
            // Need 32 command headers. Allocate extra (alignment-1) bytes for alignment.
            listAlloc := kalloc(sizeof(THBA_CMD_HEADER) * 32 + 1023);
            cmd_list_base := listAlloc;
            cmd_list_base := puint32((uint32(cmd_list_base) + 1023) and $FFFFFC00);
            memset(uint32(cmd_list_base), 0, sizeof(THBA_CMD_HEADER) * 32);

            //set the command list base address
            port^.cmdl_basel := vtop(uint32(cmd_list_base)); //todo set virtual address in device
            port^.cmdl_baseu := 0;

            device^.command_list := PHBA_CMD_HEADER(cmd_list_base);

            //allocate memory for the FIS and ensure it is aligned to 256 bytes
            fisAlloc := kalloc(sizeof(THBA_FIS) + 255);
            fis_base := fisAlloc;
            fis_base := puint32((uint32(fis_base) + 255) and $FFFFFF00);

            //set the FIS base address
            port^.fis_basel := vtop(uint32(fis_base));
            port^.fis_baseu := 0;

            memset(uint32(fis_base), 0, sizeof(THBA_FIS));

            device^.fis := PHBA_FIS(fis_base);
            //todo check how many simultaneous commands are supported

            //allocate memory for the command table and ensure it is aligned to 128 bytes
            // Need 32 command tables. Allocate extra (alignment-1) bytes for alignment.
            tableAlloc := kalloc(sizeof(THBA_CMD_TABLE) * 32 + 127);
            cmd_table_base := tableAlloc;
            cmd_table_base := puint32((uint32(cmd_table_base) + 127) and $FFFFFF80);
            memset(uint32(cmd_table_base), 0, sizeof(THBA_CMD_TABLE) * 32);

            device^.command_table := PHBA_CMD_TABLE(cmd_table_base);

            //set the command table base address and setup command table
            for ii:=0 to 31 do begin
                //set command header locations
                command_header := PHBA_CMD_HEADER(uint32(cmd_list_base) + (ii * sizeof(THBA_CMD_HEADER)));
                command_header^.prdtl := 32;
                command_header^.cmd_table_base := vtop(uint32(cmd_table_base) + (ii * sizeof(THBA_CMD_TABLE)));
                command_header^.cmd_table_baseu := 0;
            end;

            registered := false;
            for attempt := 1 to 2 do begin
                if controller^.offline or device^.offline then break;
                if not reset_port(port) then begin log_port_failure(i,port,'reset/readiness timeout'); continue; end;
                signatureWait := 0;
                while (puint32(@port^.signature)^ = 0) or (puint32(@port^.signature)^ = $FFFFFFFF) do begin
                    if signatureWait >= 500 then break;
                    psleep(16); Inc(signatureWait);
                end;
                case puint32(@port^.signature)^ of
                    SATA_SIG_ATA: begin device^.device_type := SATA; registered := identify_device(controller,i,false); end;
                    SATA_SIG_ATAPI: begin device^.device_type := ATAPI; registered := identify_device(controller,i,true); end;
                else log_port_failure(i,port,'unsupported or missing signature');
                end;
                if registered then break;
                log_port_failure(i,port,'IDENTIFY failed; retrying detection');
            end;
            if not registered then begin
                port^.int_enable := 0;
                if not controller^.offline then
                    if not stop_port(port) then fail_controller(controller,eIOTimeout);
                begin
                    port^.cmdl_basel := 0; port^.fis_basel := 0;
                    kfree(listAlloc); kfree(fisAlloc); kfree(tableAlloc);
                    device^.command_list := nil; device^.fis := nil; device^.command_table := nil;
                end;
                log_port_failure(i,port,'device unavailable after bounded retries');
            end;

            controller^.mio^.int_status := $FFFFFFFF;
        end;
    end;
end;

function identify_device(controller : PAHCI_Controller; portIndex : uint32; isATAPI : boolean) : boolean;
var
    fis         : PHBA_FIS_REG_H2D;
    cmd_header  : PHBA_CMD_HEADER;
    cmd_table   : PHBA_CMD_TABLE;
    cmd         : uint32;
    i, timeout  : uint32;
    buffer      : puint32;
    device      : PAHCI_Device;
    tfd         : uint32;
    sec_count   : uint32;
    b8          : puint16;
    storageDev  : PStorage_Device;
    storageDev_sectorSize : uint32;
    ncqDepth    : uint32;
    slotCount   : uint32;
    ncqSupported : boolean;
begin
    identify_device := false;
    device := PAHCI_Device(@controller^.devices[portIndex]);
    slotCount := ((controller^.mio^.capabilites shr 8) and $1F) + 1;
    device^.slotCount := slotCount;
    device^.supportsNCQ := false;
    device^.queueDepth := 1;

    //clear any pending interrupts
    device^.port^.int_status := $FFFFFFFF;
    device^.port^.int_enable := $0;   { disabled during IDENTIFY polling }
    
    // Allocate a 512-byte DMA buffer for the IDENTIFY d driver.storage.ctl.ide.ata
    buffer := kalloc(512);
    memset(uint32(buffer), 0, 512);

    // Use command slot 0 for the IDENTIFY command.
    cmd_header := device^.command_list;  // Assuming slot 0 is at the beginning.
    cmd_header^.cmd_fis_length := sizeof(THBA_FIS_REG_H2D) div sizeof(uint32);
    cmd_header^.wrt := 0;
    cmd_header^.prdtl := 1;
    cmd_header^.clear_busy := 1;
    cmd_header^.atapi := 0; { IDENTIFY PACKET is ATA $A1, not an ATAPI PACKET $A0 command }


    // Setup the command table (using slot 0)
    cmd_table := device^.command_table;  
    cmd_table^.prdt[0].dba := vtop(uint32(buffer));
    cmd_table^.prdt[0].dbc := 511;   // 512 bytes (0-based count)
    cmd_table^.prdt[0].int := 1;     // Interrupt on completion

    // if isATAPI then begin
    // // ATAPI Identify Device command
    // buffer[0] := $A1;  // ATAPI Identify Device
    // end;

    // Construct the Command FIS in the command table's CFIS area
    fis := PHBA_FIS_REG_H2D(@device^.command_table^.cmd_fis);
    memset(uint32(fis), 0, sizeof(THBA_FIS_REG_H2D));
    fis^.fis_type := uint8(FIS_TYPE_REG_H2D);
    fis^.c   := $1;
    if isATAPI then begin
        fis^.command  := ATA_CMD_IDENTIFY_PACKET;
    end else begin
        fis^.command  := ATA_CMD_IDENTIFY;
    end;
    fis^.device   := $40; 

    if not wait_register(@device^.port^.tfd,$88,0,2500) then begin
        log_port_failure(portIndex,device^.port,'busy before IDENTIFY'); kfree(buffer); exit;
    end;
    device^.port^.cmd_issue := 1;
    if (not wait_register(@device^.port^.cmd_issue,1,0,2500)) or
       ((device^.port^.tfd and $1) <> 0) or ((device^.port^.int_status and $40000000) <> 0) then begin
        log_port_failure(portIndex,device^.port,'IDENTIFY timeout or task-file error');
        if not stop_port(device^.port) then fail_controller(controller,eIOTimeout);
        kfree(buffer);
        exit;
    end;
    device^.port^.int_status := $FFFFFFFF;

    { Re-enable port interrupts now that IDENTIFY polling is done.
      $4000000B = bit 0 (D2H Register FIS) | bit 1 (PIO Setup FIS) | bit 3 (Set Device Bits / NCQ) | bit 30 (Task File Error)
      Bit 3 is required for NCQ completion notifications via SDB FIS. }
    device^.port^.int_enable := AHCI_IRQ_MASK;

    b8 := puint16(buffer);
    memcpy(uint32(buffer), uint32(@device^.ata_info[0]), 512);
    sec_count := (b8[61] shl 16) or b8[60];
    ncqDepth := (b8[75] and $1F) + 1;
    ncqSupported := ((b8[76] and (1 shl 8)) <> 0) and (not isATAPI) and AHCI_ENABLE_NCQ and ((controller^.mio^.capabilites and (1 shl 30)) <> 0);
    if ncqSupported then begin
        device^.supportsNCQ := true;
        device^.queueDepth := min_u32(slotCount, ncqDepth);
    end else begin
        device^.supportsNCQ := false;
        device^.queueDepth := 1;
    end;

    kfree(buffer);

    { For ATAPI devices, get capacity from SCSI READ CAPACITY instead of IDENTIFY }
    if isATAPI then begin
        if send_read_capacity(device, @sec_count, @storageDev_sectorSize) then begin
        end else begin
            io.syslog.writestringln('driver.storage.ctl.ahci: ATAPI capacity query failed (no media?).');
            storageDev_sectorSize := 2048;
            sec_count := 0;
        end;
    end else begin
        storageDev_sectorSize := 512;
    end;
    
    { Register this device with the storage manager.
      ATAPI devices (optical drives) are always registered even with 0 sectors (no media). }
    if (sec_count > 0) or isATAPI then begin
        storageDev := PStorage_Device(kalloc(sizeof(TStorage_Device)));
        memset(uint32(storageDev), 0, sizeof(TStorage_Device));

        storageDev^.controllerId0  := uint32(device);  { store PAHCI_Device for callback wrappers }
        storageDev^.sectorSize     := storageDev_sectorSize;
        storageDev^.maxSectorCount := sec_count;
        storageDev^.start          := 0;
        storageDev^.volumes        := DL_New(sizeof(Pointer));

        if isATAPI then begin
            storageDev^.controller := TControllerType.ControllerAHCI_ATAPI;
            storageDev^.writable   := false;
            storageDev^.dispatchRead       := TDriverDispatch(@ahci_dispatch_atapi_read);
            storageDev^.dispatchWrite      := TDriverDispatch(nil);
            storageDev^.maxActive          := 1;
        end else begin
            storageDev^.controller := TControllerType.ControllerAHCI;
            storageDev^.writable   := true;
            storageDev^.dispatchRead       := TDriverDispatch(@ahci_dispatch_read);
            storageDev^.dispatchWrite      := TDriverDispatch(@ahci_dispatch_write);
            storageDev^.maxActive          := device^.queueDepth;
        end;

        driver.storage.mgr.register_device(storageDev);
        kfree(void(storageDev)); { manager owns a copy }
        identify_device := true;

        io.syslog.writestring('driver.storage.ctl.ahci: Registered device on port ');
        io.syslog.writeint(portIndex);
        if sec_count > 0 then begin
            io.syslog.writestring(' with storage manager (');
            io.syslog.writeint((uint64(sec_count) * storageDev_sectorSize) div 1024 div 1024);
            io.syslog.writestringln(' MB)');
        end else begin
            io.syslog.writestringln(' with storage manager (no media)');
        end;
    end else begin
        io.syslog.writestring('driver.storage.ctl.ahci: Port ');
        io.syslog.writeint(portIndex);
        io.syslog.writestringln(': sector count is 0, skipping registration.');
    end;



end;


// procedure controller_reset(controller : PAHCI_Controller);
// begin
//     //check sam TODO
//     if (controller^.mio^.capabilites2 and $1) = 0 then begin
//         io.syslog.writestringln('driver.storage.ctl.ahci: Controller does not support reset');
//         exit;
//     end;

//     //check ghc.ae is set to 1
//     if (controller^.mio^.global_ctrl and $1) = 0 then begin
//         io.syslog.writestringln('driver.storage.ctl.ahci: Controller is not enabled');
//         controller^.mio^.global_ctrl := controller^.mio^.global_ctrl or $1;
//     end;

//     while (controller^.mio^.global_ctrl and $1) <> 0 do begin
//         //wait for the controller to be ready
//     end;

//     //set the reset bit
//     controller^.mio^.global_ctrl := controller^.mio^.global_ctrl or $2;



// end;

function load(ptr : void) : boolean;
var 
    device : PPCI_Device;
    controller : PAHCI_Controller;
    i : uint32;
    base : PHBA_Memory;
    cmd_list_base : puint32;
    fis_base : puint32;
    cmd_table_base : puint32;
    int_no : uInt8;
    timeout : uint32;
    flags : uint32;
begin
    if ahciControllers = nil then begin
        ahciControllers := DL_New(SizeOf(Pointer));
    end;

    device := PPCI_Device(ptr);

    driver.bus.pci.enableDevice(device^.bus, device^.slot, device^.func);
    psleep(5);
    int_no := device^.interrupt_line + 32;

    load := false;
    controller := PAHCI_Controller(kalloc(sizeof(TAHCI_Controller)));
    if controller = nil then exit;

    controller^.pci_device := PPCI_Device(device);
    
//     Perform BIOS/OS handoff (if the bit in the extended capabilities is set)
//     Reset controller

    //get the base address of the controller
    page_base := kpalloc(device^.address5); // TODO MEM memory manager need to be improved
    base := PHBA_Memory(device^.address5);
    controller^.mio := base;

    if (base^.capabilites2 and 1) <> 0 then begin
        base^.bohc := base^.bohc or 2;
        if not wait_register(@base^.bohc,$11,0,1000) then begin
            io.syslog.logln('AHCI','BIOS handoff timed out; controller not started.');
            kfree(void(controller)); exit;
        end;
    end;
    base^.global_ctrl := base^.global_ctrl or AHCI_CONTROLLER_MODE or 1;
    if not wait_register(@base^.global_ctrl,1,0,500) then begin
        io.syslog.logln('AHCI','Controller reset timed out; controller not started.');
        driver.bus.pci.setBusMaster(device^.bus,device^.slot,device^.func,false);
        kfree(void(controller)); exit;
    end;
    { Publish a stable heap pointer, never an interior pointer into a growing DList. }
    asm pushfd; cli; pop flags end;
    DL_Add(ahciControllers); DL_Set(ahciControllers,DL_Size(ahciControllers)-1,puint32(@controller));
    if (flags and $200) <> 0 then asm sti end;

    registerISR(int_no, @ahci_isr);

    io.syslog.writestring('driver.storage.ctl.ahci: IRQ ');
    io.syslog.writeint(device^.interrupt_line);
    io.syslog.writestring(' -> INT ');
    io.syslog.writeint(int_no);
    io.syslog.writestringln('');

    { Initialise IOAPIC/LAPIC support (safe no-op if no IOAPIC present).
      driver.bus.pci interrupts may be routed through the IOAPIC instead of the
      legacy 8259 PIC — without programming the IOAPIC redirection
      entry the interrupt never reaches the CPU. }
    arch.x86.isr.ioapic.init();
    arch.x86.isr.ioapic.route_pci_irq(device^.interrupt_line, int_no);

    //enable driver.storage.ctl.ahci mode
    base^.global_ctrl := base^.global_ctrl or AHCI_CONTROLLER_MODE;

    //enable interrupts (GHC.IE = bit 1)
    base^.global_ctrl := base^.global_ctrl or $2;

    //clear any pending interrupts
    base^.int_status := $FFFFFFFF;

    check_ports(controller);
    load := not controller^.offline;
end;

function now_tick : uint32;
begin now_tick := puint32(@Counters.c32)^; end;

{ Only complete commands whose DMA ownership has ended. Callback re-entry may
  submit more work, so consume a fixed mask rather than rescanning new slots. }
procedure finish_slots(device : PAHCI_Device; mask : uint32; error : TError);
var slot : uint32; cb : TIOCompletion; data : puint32;
begin
    for slot := 0 to 31 do if (mask and (uint32(1) shl slot)) <> 0 then begin
        if device^.pending[slot].inUse and device^.pending[slot].issued then begin
            cb := device^.pending[slot].completion; data := device^.pending[slot].userdata;
            device^.pending[slot].inUse := false; device^.pending[slot].issued := false;
            device^.pending[slot].completion := nil; device^.pending[slot].userdata := nil;
            if cb <> nil then cb(error,data);
        end;
    end;
end;

procedure fail_controller(controller : PAHCI_Controller; error : TError);
var j, command : uint32; pci : PPCI_Device; device : PAHCI_Device;
begin
    if controller = nil then exit;
    controller^.offline := true;
    pci := controller^.pci_device;
    { Last resort: fence DMA for the whole controller before returning buffers.
      Write only PCI command bits, not the adjacent write-one-to-clear status. }
    driver.bus.pci.requestConfig(pci^.bus,pci^.slot,pci^.func,1);
    command := inl($CFC) and $FFFF;
    driver.bus.pci.writeConfig(pci^.bus,pci^.slot,pci^.func,1,command and not uint32(4));
    driver.bus.pci.requestConfig(pci^.bus,pci^.slot,pci^.func,1);
    command := inl($CFC);
    if (command <> $FFFFFFFF) and ((command and 4) <> 0) then
        core.panic.panic('AHCI DMA cannot be stopped','PCI bus mastering could not be disabled',nil);
    controller^.mio^.global_ctrl := controller^.mio^.global_ctrl and not uint32(2);
    { Reject all new work before any callback drains a queued request. }
    for j := 0 to 31 do controller^.devices[j].offline := true;
    for j := 0 to 31 do begin
        device := @controller^.devices[j];
        device^.recovering := false;
        if device^.port <> nil then begin
            device^.port^.int_enable := 0;
            device^.port^.cmd := device^.port^.cmd and not uint32($11);
        end;
        finish_slots(device,$FFFFFFFF,error);
    end;
    io.syslog.logln('AHCI','Controller taken offline after DMA stop timeout.');
end;

procedure begin_recovery(device : PAHCI_Device; error : TError);
begin
    if device^.recovering or device^.offline then exit;
    device^.offline := true; device^.recovering := true;
    device^.recoveryTick := now_tick; device^.recoveryError := error;
    device^.port^.int_enable := 0;
    device^.port^.cmd := device^.port^.cmd and not uint32(1);
    io.syslog.logln('AHCI','I/O failed or timed out; stopping port before releasing DMA buffers.');
end;

{ Nonblocking; called by IRQ and the timer watchdog with interrupts disabled. }
procedure service_port(device : PAHCI_Device; now, status : uint32);
var active, doneMask, pendingMask, slot : uint32; expired : boolean;
begin
    if device^.port = nil then exit;
    if device^.recovering then begin
        if (puint32(@device^.port^.cmd)^ and $8000) = 0 then begin
            device^.port^.cmd := device^.port^.cmd and not uint32($10);
            if (puint32(@device^.port^.cmd)^ and $4000) = 0 then begin
                device^.recovering := false;
                device^.port^.int_status := $FFFFFFFF;
                finish_slots(device,$FFFFFFFF,device^.recoveryError);
                exit;
            end;
        end;
        if uint32(now-device^.recoveryTick) >= AHCI_STOP_TICKS then
            fail_controller(PAHCI_Controller(device^.owner),device^.recoveryError);
        exit;
    end;
    if device^.offline then exit;
    active := puint32(@device^.port^.cmd_issue)^ or puint32(@device^.port^.sata_active)^;
    doneMask := 0; pendingMask := 0; expired := false;
    for slot := 0 to 31 do if device^.pending[slot].inUse and device^.pending[slot].issued then begin
        pendingMask := pendingMask or (uint32(1) shl slot);
        if (active and (uint32(1) shl slot)) = 0 then doneMask := doneMask or (uint32(1) shl slot)
        else if uint32(now-device^.pending[slot].started) >= AHCI_IO_TICKS then expired := true;
    end;
    if pendingMask = 0 then exit; { polling IDENTIFY owns its own recovery }
    if (status and AHCI_FATAL_MASK) <> 0 then begin
        begin_recovery(device,eIOError); exit;
    end;
    if (doneMask <> 0) and ((device^.port^.tfd and $1) <> 0) then begin
        begin_recovery(device,eIOError); exit;
    end;
    if expired then begin begin_recovery(device,eIOTimeout); exit; end;
    finish_slots(device,doneMask,eNone);
end;

procedure ahci_isr();
var i,j,ctrlIS,portIS : uint32; controller : PAHCI_Controller; device : PAHCI_Device;
begin
    if (ahciControllers = nil) or (DL_Size(ahciControllers) = 0) then exit;
    for i := 0 to DL_Size(ahciControllers)-1 do begin
        controller := PAHCI_Controller(puint32(DL_Get(ahciControllers,i))^);
        if controller^.offline then continue;
        ctrlIS := controller^.mio^.int_status;
        if ctrlIS = 0 then continue;
        { Acknowledge before callbacks can submit a new command/interrupt. }
        for j := 0 to 31 do if (ctrlIS and (uint32(1) shl j)) <> 0 then begin
            device := @controller^.devices[j];
            if device^.port = nil then continue;
            portIS := device^.port^.int_status;
            device^.port^.int_status := portIS;
            controller^.mio^.int_status := uint32(1) shl j;
            service_port(device,now_tick,portIS);
        end;
    end;
end;

procedure ahci_watchdog(data : void);
var i,j,now : uint32; controller : PAHCI_Controller;
begin
    Inc(watchdogDivider); if watchdogDivider < 80 then exit; watchdogDivider := 0;
    if (ahciControllers = nil) or (DL_Size(ahciControllers) = 0) then exit;
    now := now_tick;
    for i := 0 to DL_Size(ahciControllers)-1 do begin
        controller := PAHCI_Controller(puint32(DL_Get(ahciControllers,i))^);
        if controller^.offline then continue;
        for j := 0 to 31 do
            if controller^.devices[j].slotCount <> 0 then service_port(@controller^.devices[j],now,controller^.devices[j].port^.int_status);
    end;
end;

{ Atomic reserve: software ownership precedes descriptor construction. }
function find_cmd_slot(device : PAHCI_Device) : uint32;
var flags, slot, active, count : uint32;
begin
    find_cmd_slot := $FFFFFFFF;
    if (device = nil) or (device^.port = nil) then exit;
    asm pushfd; cli; pop flags end;
    if not device^.offline then begin
        active := device^.port^.cmd_issue or device^.port^.sata_active;
        count := device^.slotCount; if count > 32 then count := 32;
        if not device^.supportsNCQ then if count > 1 then count := 1;
        if count > 0 then for slot := 0 to count-1 do begin
            if ((active and (uint32(1) shl slot)) = 0) and (not device^.pending[slot].inUse) then begin
                device^.pending[slot].inUse := true; device^.pending[slot].issued := false;
                find_cmd_slot := slot; break;
            end;
        end;
    end;
    if (flags and $200) <> 0 then asm sti end;
end;

{ Split at the kernel's 4 MiB virtual mapping boundaries. Every segment gets
  its own physical translation, including buffers with an unaligned start. }
function build_prdt(table : PHBA_CMD_TABLE; address, bytes : uint32; var entries : uint32) : TError;
var remaining, chunk : uint32;
begin
    build_prdt := eInvalidArgument; entries := 0;
    if (bytes = 0) or ((address and 1) <> 0) or ((bytes and 1) <> 0) or (bytes > $FFFFFFFF-address) then exit;
    remaining := bytes;
    while remaining > 0 do begin
        if entries = 32 then exit;
        chunk := $400000-(address and $3FFFFF); if chunk > remaining then chunk := remaining;
        table^.prdt[entries].dba := vtop(address); table^.prdt[entries].dbau := 0;
        table^.prdt[entries].rsv0 := 0; table^.prdt[entries].reserved := 0;
        table^.prdt[entries].dbc := chunk-1; table^.prdt[entries].int := 0;
        Inc(entries); address := address+chunk; remaining := remaining-chunk;
    end;
    table^.prdt[entries-1].int := 1;
    build_prdt := eNone;
end;

{ mode: 0 SATA read, 1 SATA write, 2 ATAPI READ(12), 3 READ CAPACITY. }
function submit_command(device : PAHCI_Device; lba : uint64; bytes : uint32; buffer : puint32;
                        mode : uint32; completion : TIOCompletion; userdata : puint32) : TError;
var slot, entries, sectors, flags : uint32; header : PHBA_CMD_HEADER; table : PHBA_CMD_TABLE;
    fis : PHBA_FIS_REG_H2D; err : TError;
begin
    submit_command := eInvalidArgument;
    if (device = nil) or (buffer = nil) or (bytes = 0) then exit;
    if (device^.port = nil) or (device^.command_list = nil) or (device^.command_table = nil) then exit;
    if device^.offline then begin submit_command := eDeviceNotReady; exit; end;
    if mode < 2 then begin
        if ((bytes mod 512) <> 0) or (bytes > 33554432) or (lba > $FFFFFFFFFFFF) then exit;
        sectors := bytes div 512;
        if uint64(sectors-1) > $FFFFFFFFFFFF-lba then exit;
    end else if mode = 2 then begin
        if ((bytes mod 2048) <> 0) or (lba > $FFFFFFFF) then exit;
        sectors := bytes div 2048;
        if uint64(sectors-1) > $FFFFFFFF-lba then exit;
    end else begin
        if (mode <> 3) or (bytes <> 8) then exit;
        sectors := 0;
    end;
    slot := find_cmd_slot(device);
    if slot = $FFFFFFFF then begin submit_command := eNoFreeSlot; exit; end;
    header := @device^.command_list[slot]; table := @device^.command_table[slot];
    { Preserve the command-table pointer but reset every per-command header flag. }
    puint32(header)^ := 0; header^.prdbc := 0;
    header^.cmd_fis_length := sizeof(THBA_FIS_REG_H2D) div 4;
    if mode = 1 then header^.wrt := 1;
    if mode >= 2 then header^.atapi := 1;
    memset(uint32(table),0,128);
    err := build_prdt(table,uint32(buffer),bytes,entries);
    if err = eNone then begin
        header^.prdtl := entries; fis := PHBA_FIS_REG_H2D(@table^.cmd_fis);
        fis^.fis_type := uint8(FIS_TYPE_REG_H2D); fis^.c := 1;
        if mode < 2 then begin
            fis^.device := $40;
            fis^.lba0 := lba and $FF; fis^.lba1 := (lba shr 8) and $FF;
            fis^.lba2 := (lba shr 16) and $FF; fis^.lba3 := (lba shr 24) and $FF;
            fis^.lba4 := (lba shr 32) and $FF; fis^.lba5 := (lba shr 40) and $FF;
            if device^.supportsNCQ then begin
                if mode = 0 then fis^.command := ATA_CMD_READ_FPDMA_QUEUED else fis^.command := ATA_CMD_WRITE_FPDMA_QUEUED;
                fis^.featurel := sectors and $FF; fis^.featureh := (sectors shr 8) and $FF;
                fis^.countl := slot shl 3;
            end else begin
                if mode = 0 then fis^.command := ATA_CMD_READ_DMA_EXT else fis^.command := ATA_CMD_WRITE_DMA_EXT;
                fis^.countl := sectors and $FF; fis^.counth := (sectors shr 8) and $FF;
            end;
        end else begin
            fis^.command := ATA_CMD_PACKET; fis^.device := $A0; fis^.featurel := 1;
            if mode = 3 then table^.acmd[0] := $25
            else begin
                table^.acmd[0] := $A8;
                table^.acmd[2] := (lba shr 24) and $FF; table^.acmd[3] := (lba shr 16) and $FF;
                table^.acmd[4] := (lba shr 8) and $FF; table^.acmd[5] := lba and $FF;
                table^.acmd[6] := (sectors shr 24) and $FF; table^.acmd[7] := (sectors shr 16) and $FF;
                table^.acmd[8] := (sectors shr 8) and $FF; table^.acmd[9] := sectors and $FF;
            end;
        end;
    end;
    asm pushfd; cli; pop flags end;
    if err = eNone then begin
        if device^.offline then err := eDeviceNotReady
        else if (device^.port^.tfd and $88) <> 0 then err := eDeviceNotReady;
    end;
    if err = eNone then begin
        device^.pending[slot].completion := completion; device^.pending[slot].userdata := userdata;
        device^.pending[slot].started := now_tick; device^.pending[slot].issued := true;
        { RW1 registers: never write a stale snapshot of other command bits. }
        if (mode < 2) and device^.supportsNCQ then device^.port^.sata_active := uint32(1) shl slot;
        device^.port^.cmd_issue := uint32(1) shl slot;
    end else device^.pending[slot].inUse := false;
    if (flags and $200) <> 0 then asm sti end;
    submit_command := err;
end;

function send_read_dma_async(device : PAHCI_Device; lba : uint64; count : uint32; buffer : puint32; completion : TIOCompletion; userdata : puint32) : TError;
begin send_read_dma_async := submit_command(device,lba,count,buffer,0,completion,userdata); end;
function send_write_dma_async(device : PAHCI_Device; lba : uint64; count : uint32; buffer : puint32; completion : TIOCompletion; userdata : puint32) : TError;
begin send_write_dma_async := submit_command(device,lba,count,buffer,1,completion,userdata); end;
function read_atapi_async(device : PAHCI_Device; lba : uint64; count : uint32; buffer : puint32; completion : TIOCompletion; userdata : puint32) : TError;
begin read_atapi_async := submit_command(device,lba,count,buffer,2,completion,userdata); end;

type TCapacityWait = record Done : uint32; Error : TError; end;
     PCapacityWait = ^TCapacityWait;
procedure capacity_done(error : TError; userdata : puint32);
var state : PCapacityWait;
begin state := PCapacityWait(userdata); state^.Error := error; puint32(@state^.Done)^ := 1; end;

function send_read_capacity(device : PAHCI_Device; sectorCount : puint32; blockSize : puint32) : boolean;
var buffer : puint32; state : TCapacityWait; rawLBA, rawBlkSz : uint32; err : TError;
begin
    send_read_capacity := false; sectorCount^ := 0; blockSize^ := 0;
    if not INTE then exit; { init/task context only }
    buffer := kalloc(512); if buffer = nil then exit;
    memset(uint32(buffer),0,512); state.Done := 0; state.Error := eNone;
    err := submit_command(device,0,8,buffer,3,@capacity_done,puint32(@state));
    if err = eNone then begin
        { The watchdog ends ownership only after stopping DMA on error/timeout. }
        while puint32(@state.Done)^ = 0 do asm hlt end;
        err := state.Error;
    end;
    if err = eNone then begin
        rawLBA := buffer[0]; rawBlkSz := buffer[1];
        rawLBA := ((rawLBA and $FF) shl 24) or ((rawLBA and $FF00) shl 8) or
                  ((rawLBA and $FF0000) shr 8) or ((rawLBA and $FF000000) shr 24);
        rawBlkSz := ((rawBlkSz and $FF) shl 24) or ((rawBlkSz and $FF00) shl 8) or
                    ((rawBlkSz and $FF0000) shr 8) or ((rawBlkSz and $FF000000) shr 24);
        if (rawLBA <> $FFFFFFFF) and (rawBlkSz = 2048) then begin
            sectorCount^ := rawLBA+1; blockSize^ := rawBlkSz; send_read_capacity := true;
        end;
    end;
    kfree(buffer);
end;

{ ---------- StorageManager async callback wrappers ---------- }

{ ========================================================================== }
{              Phase 3: TDriverDispatch bridge to driver.storage.ctl.ahci DMA                   }
{ ========================================================================== }

{ ISR-safe completion — bridges from per-slot TIOCompletion callback to
  driver.storage.mgr.complete_io.  userdata is the PIORequest pointer. }
procedure ahci_io_completion(error : TError; userdata : puint32);
var request : PIORequest;
begin
    request := PIORequest(userdata);
    if error = eNone then request^.ByteCount := request^.SectorCount*request^.Device^.sectorSize;
    driver.storage.mgr.complete_io(request,error = eNone,error);
end;

{ TDriverDispatch — SATA read via DMA }
procedure ahci_dispatch_read(device : PStorage_Device; request : PIORequest);
var
    ahciDev : PAHCI_Device;
    err     : TError;
begin
    if (device^.sectorSize = 0) or (request^.SectorCount = 0) or
       (request^.SectorCount > $FFFFFFFF div device^.sectorSize) or
       (request^.LBA >= device^.maxSectorCount) or
       (request^.SectorCount > device^.maxSectorCount-request^.LBA) then begin
        driver.storage.mgr.complete_io(request,false,eInvalidArgument); exit;
    end;
    ahciDev := PAHCI_Device(device^.controllerId0);
    err := send_read_dma_async(ahciDev, request^.LBA,
            request^.SectorCount * device^.sectorSize,
            puint32(request^.Buffer),
            @ahci_io_completion, puint32(request));
    if err <> eNone then
        driver.storage.mgr.complete_io(request, false, err);
end;

{ TDriverDispatch — SATA write via DMA }
procedure ahci_dispatch_write(device : PStorage_Device; request : PIORequest);
var
    ahciDev : PAHCI_Device;
    err     : TError;
begin
    if (device^.sectorSize = 0) or (request^.SectorCount = 0) or
       (request^.SectorCount > $FFFFFFFF div device^.sectorSize) or
       (request^.LBA >= device^.maxSectorCount) or
       (request^.SectorCount > device^.maxSectorCount-request^.LBA) then begin
        driver.storage.mgr.complete_io(request,false,eInvalidArgument); exit;
    end;
    ahciDev := PAHCI_Device(device^.controllerId0);
    err := send_write_dma_async(ahciDev, request^.LBA,
            request^.SectorCount * device^.sectorSize,
            puint32(request^.Buffer),
            @ahci_io_completion, puint32(request));
    if err <> eNone then
        driver.storage.mgr.complete_io(request, false, err);
end;

{ TDriverDispatch — ATAPI read via SCSI READ(12) }
procedure ahci_dispatch_atapi_read(device : PStorage_Device; request : PIORequest);
var
    ahciDev : PAHCI_Device;
    err     : TError;
begin
    if (device^.sectorSize = 0) or (request^.SectorCount = 0) or
       (request^.SectorCount > $FFFFFFFF div device^.sectorSize) or
       (request^.LBA >= device^.maxSectorCount) or
       (request^.SectorCount > device^.maxSectorCount-request^.LBA) then begin
        driver.storage.mgr.complete_io(request,false,eInvalidArgument); exit;
    end;
    ahciDev := PAHCI_Device(device^.controllerId0);
    err := read_atapi_async(ahciDev, request^.LBA,
            request^.SectorCount * device^.sectorSize,
            puint32(request^.Buffer),
            @ahci_io_completion, puint32(request));
    if err <> eNone then
        driver.storage.mgr.complete_io(request, false, err);
end;


{ Synthetic MMIO only: these tests never issue I/O to a real disk. }
type TTestCompletion = record Count : uint32; Error : TError; end;
     PTestCompletion = ^TTestCompletion;
procedure test_done(error : TError; data : puint32);
begin
    Inc(PTestCompletion(data)^.Count); PTestCompletion(data)^.Error := error;
end;

procedure run_ahci_tests(stdout_buf : POutBuf);
var device : PAHCI_Device; port : PHBA_Port; table : PHBA_CMD_TABLE;
    buffer : pointer; aligned, entries, first, second, passed, failed : uint32;
    result : TError; completed : TTestCompletion; msg, number : pchar;
    procedure Assert(ok : boolean; name : pchar);
    begin
        if ok then Inc(passed) else begin
            Inc(failed); io.syslog.log('AHCITEST','FAIL: '); io.syslog.writestringln(name);
            if stdout_buf <> nil then begin
                io.stdio.bufWriteStr(stdout_buf,'FAIL: '); io.stdio.bufWriteStrLn(stdout_buf,name);
            end;
        end;
    end;
    procedure reset_fake;
    begin
        memset(uint32(port),0,sizeof(THBA_Port));
        memset(uint32(@device^.pending),0,sizeof(device^.pending));
        device^.offline := false; device^.recovering := false;
        device^.supportsNCQ := true; completed.Count := 0; completed.Error := eNone;
    end;
begin
    passed := 0; failed := 0;
    device := PAHCI_Device(kalloc(sizeof(TAHCI_Device)));
    port := PHBA_Port(kalloc(sizeof(THBA_Port))); device^.port := port;
    device^.command_list := PHBA_CMD_HEADER(kalloc(32*sizeof(THBA_CMD_HEADER)));
    device^.command_table := PHBA_CMD_TABLE(kalloc(32*sizeof(THBA_CMD_TABLE)));
    device^.slotCount := 32; table := device^.command_table;
    buffer := kalloc($800000); aligned := (uint32(buffer)+$3FFFFF) and $FFC00000;
    reset_fake;
    Assert(build_prdt(table,aligned,$400000,entries)=eNone,'exact 4 MiB accepted');
    Assert((entries=1) and (table^.prdt[0].dbc=$3FFFFF),'exact 4 MiB has one nonempty PRD');
    Assert(build_prdt(table,aligned-512,1024,entries)=eNone,'cross-page transfer accepted');
    Assert((entries=2) and (table^.prdt[0].dbc=511) and (table^.prdt[1].dbc=511),'cross-page segments bounded');
    Assert((table^.prdt[0].dba=vtop(aligned-512)) and (table^.prdt[1].dba=vtop(aligned)),'each page translated independently');
    Assert((table^.prdt[0].int=0) and (table^.prdt[1].int=1),'only final PRD interrupts');
    Assert(build_prdt(table,aligned,0,entries)=eInvalidArgument,'zero length rejected');
    Assert(build_prdt(table,aligned+1,512,entries)=eInvalidArgument,'odd address rejected');
    Assert(build_prdt(table,aligned,513,entries)=eInvalidArgument,'odd length rejected');
    Assert(build_prdt(table,$FFFFFF00,512,entries)=eInvalidArgument,'address wrap rejected');
    first := find_cmd_slot(device); second := find_cmd_slot(device);
    Assert((first=0) and (second=1),'reservation excludes concurrent builder');
    service_port(device,now_tick,AHCI_FATAL_MASK);
    Assert(device^.pending[0].inUse and not device^.offline,'unissued reservation not completed or recovered');
    reset_fake; device^.supportsNCQ := false;
    first := find_cmd_slot(device); second := find_cmd_slot(device);
    Assert((first=0) and (second=$FFFFFFFF),'non-NCQ serializes commands');
    reset_fake; port^.cmd_issue := $20; port^.sata_active := $20;
    result := send_read_dma_async(device,0,512,puint32(buffer),@test_done,puint32(@completed));
    Assert(result=eNone,'valid read accepted');
    Assert((port^.cmd_issue=1) and (port^.sata_active=1),'issue writes only newly set bit');
    Assert(device^.pending[0].issued,'ownership published before issuing');
    port^.cmd_issue := 0; port^.sata_active := 0;
    service_port(device,now_tick,0); service_port(device,now_tick,0);
    Assert((completed.Count=1) and (completed.Error=eNone),'successful completion exactly once');
    reset_fake;
    Assert(send_read_dma_async(device,0,513,puint32(buffer),nil,nil)=eInvalidArgument,'partial sector rejected');
    Assert(send_read_dma_async(device,0,33554944,puint32(buffer),nil,nil)=eInvalidArgument,'ATA count overflow rejected');
    Assert(send_read_dma_async(device,$FFFFFFFFFFFF,1024,puint32(buffer),nil,nil)=eInvalidArgument,'ATA LBA wrap rejected');
    Assert(read_atapi_async(device,$FFFFFFFF,4096,puint32(buffer),nil,nil)=eInvalidArgument,'ATAPI LBA wrap rejected');
    Assert((port^.cmd_issue=0) and not device^.pending[0].inUse,'invalid requests do not acquire DMA ownership');
    port^.tfd := $80;
    Assert(send_read_dma_async(device,0,512,puint32(buffer),nil,nil)=eDeviceNotReady,'busy device fails without spinning');
    Assert(not device^.pending[0].inUse,'rejected command releases reservation');
    reset_fake;
    result := send_read_dma_async(device,0,512,puint32(buffer),@test_done,puint32(@completed));
    port^.cmd := $C011;
    service_port(device,now_tick,$40000000);
    Assert(device^.recovering and (completed.Count=0),'task-file error begins recovery without releasing buffer');
    service_port(device,device^.recoveryTick,0);
    Assert((completed.Count=0) and device^.pending[0].inUse,'CR retains DMA ownership');
    port^.cmd := $4010; service_port(device,device^.recoveryTick,0);
    Assert((completed.Count=0) and ((port^.cmd and $10)=0),'FR retains DMA ownership after FRE cleared');
    port^.cmd := 0; service_port(device,device^.recoveryTick,0); service_port(device,now_tick,0);
    Assert((completed.Count=1) and (completed.Error=eIOError),'stopped error completes exactly once');
    Assert(send_read_dma_async(device,0,512,puint32(buffer),nil,nil)=eDeviceNotReady,'failed port rejects new commands');
    reset_fake;
    result := submit_command(device,0,8,puint32(buffer),3,@test_done,puint32(@completed));
    Assert((result=eNone) and (table^.acmd[0]=$25),'READ CAPACITY uses tracked ownership');
    device^.pending[0].started := 0; port^.cmd := $C011;
    service_port(device,AHCI_IO_TICKS,0);
    Assert(device^.recovering and (completed.Count=0),'watchdog retains timed-out capacity buffer');
    port^.cmd := 0; service_port(device,device^.recoveryTick,0);
    Assert((completed.Count=1) and (completed.Error=eIOTimeout),'timeout completes after DMA stop');
    port^.cmd := 1;
    Assert(not wait_register(@port^.cmd,1,0,0),'stuck reset exits at bounded deadline');
    port^.cmd := 0;
    Assert(wait_register(@port^.cmd,1,0,0),'cleared reset succeeds');
    kfree(buffer); kfree(void(device^.command_table)); kfree(void(device^.command_list)); kfree(void(port)); kfree(void(device));
    number := intToString(passed); msg := stringConcat(number,' passed, '); kfree(void(number));
    number := intToString(failed); buffer := msg; msg := stringConcat(msg,number); kfree(buffer); kfree(void(number));
    buffer := msg; msg := stringConcat(msg,' failed'); kfree(buffer);
    io.syslog.logln('AHCITEST',msg);
    if stdout_buf <> nil then io.stdio.bufWriteStrLn(stdout_buf,msg);
    kfree(void(msg));
end;

procedure UnitTest;
begin run_ahci_tests(nil); end;
procedure cmd_ahcitest(params : PParamList; stdin_buf, stdout_buf, stderr_buf : POutBuf);
begin run_ahci_tests(stdout_buf); end;

Initialization
    boot.mgr.registerBoot('driver.storage.ctl.ahci', @init, 'AHCI Controller', 'driver.storage.fs.*');

end.