Merge pull request 'Storage system overhaul: VFS, FAT32, ISO9660, AHCI/IDE refactor, GUI apps & boot drive mounting' (#49) from feature/storage-system into develop

Reviewed-on: #49
Reviewed-by: admin <[email protected]>

.gitignore

@@ -14,5 +14,6 @@ localenv.json
 /lvgl/
 dockerout.txt
 AGENTS.md
+lessons_learnt.md
 *.log
 /doc/*.md

compile_sources.sh

@@ -4,4 +4,10 @@ echo "======================="
 echo " "
 echo "Compiling FPC Sources..."
 echo " "
-fpc -Aelf -gw -g -gl -n -v0ew -O3 -OpPENTIUM3 -Si -Sc -Sg -Xd -CX -XXs -CfSSE -CfSSE2 -Rintel -Pi386 -Tlinux -FElib/ -Fusrc/* -Fusrc/include/* -Fusrc/driver/* -Fusrc/driver/net/* -Fusrc/driver/bus/* -Fusrc/driver/bus/usb/* -Fusrc/driver/hid/* src/kernel.pas
+# Build -Fu flags for all directories under src/ and wasuro/
+FU_PATHS="-Fusrc -Fuwasuro"
+for dir in $(find src wasuro -type d 2>/dev/null); do
+    FU_PATHS="$FU_PATHS -Fu$dir"
+done
+
+fpc -Aelf -gw -g -gl -n -v0ew -O3 -OpPENTIUM3 -Si -Sc -Sg -Xd -CX -XXs -CfSSE -CfSSE2 -Rintel -Pi386 -Tlinux -FElib/ $FU_PATHS src/kernel.pas

doc/flatfs.md

@@ -0,0 +1,32 @@
+#Flat filesystem
+
+A super simple filesystem for asuro. Folders are emulated in filenames.
+
+Starts with disk info sector, sector 0 of volume
+
+---
+#### disk info
+
+jmp2boot        : ubit24;
+OEMName         : array[0..7] of char;
+version         : uint16 // numerical version of filesystem
+sectorCount      : uint16;
+fileCount       : uint16
+signature       : uint32 = 0x0B00B1E5
+
+
+the Rest of the sector is reserved
+
+---
+
+Starting from sector 1 is the file table. Table size is determined by entry size (64) * fileCount
+
+---
+####File entry
+
+name : array[0..59] of char //file name max 60 chars
+fileStart : 16bit // start sector of data
+fileSize : 16bit // data size in sectors
+
+---
+

iso/hello.txt

@@ -0,0 +1 @@
+Hello from the Asuro ISO!

src/driver/bus/PCI.pas

@@ -91,6 +91,7 @@ function getDeviceInfo(class_code : uint8; subclass_code : uint8; prog_if : uint
 procedure requestConfig(bus : uint8; slot : uint8; func : uint8; row : uint8);
 procedure writeConfig(bus: uint8; slot : uint8; func : uint8; row : uint8; val : uint32);
 procedure setBusMaster(bus : uint8; slot : uint8; func : uint8; master : boolean);
+procedure enableDevice(bus : uint8; slot : uint8; func : uint8);
 
 implementation 
 
@@ -442,4 +443,34 @@ begin
     pop_trace;
 end;
 
+//Enable device inturrupts and set bus master
+procedure enableDevice(bus : uint8; slot : uint8; func : uint8);
+var
+    addr : uint32;
+    cmd : uint32;
+begin
+    push_trace('PCI.enableDevice');
+
+    addr := ($1 shl 31);
+    addr := addr or (bus shl 16);
+    addr := addr or ((slot) shl 11);
+    addr := addr or ((func) shl 8);
+    addr := addr or ($04 and $FC);
+
+    outl(PCI_CONFIG_ADDRESS_PORT, addr);
+    cmd := inl(PCI_CONFIG_DATA_PORT);
+    cmd := cmd or PCI_COMMAND_MEM_SPACE;
+    cmd := cmd or PCI_COMMAND_BUS_MASTER;
+    cmd := cmd or (3 shl 1);
+
+    //enable interrupts, remove disable interrupt bit maybe
+    cmd := cmd and not PCI_COMMAND_INT_DISABLE;
+    // cmd := cmd and not (1 shl 9);
+
+    outl(PCI_CONFIG_ADDRESS_PORT, addr);
+    outl(PCI_CONFIG_DATA_PORT, cmd);
+
+    pop_trace;
+end;
+
 end.

src/driver/include/drivertypes.pas

@@ -21,6 +21,29 @@ unit drivertypes;
 
 interface
 
+const
+    PCI_CONFIG_ADDRESS_PORT = $0CF8;
+    PCI_CONFIG_DATA_PORT    = $0CFC;
+
+    PCI_COMMAND_IO_SPACE    = $0001;
+    PCI_COMMAND_MEM_SPACE   = $0002;
+    PCI_COMMAND_BUS_MASTER  = $0004;
+    PCI_COMMAND_SPECIAL_CYC = $0008;
+    PCI_COMMAND_MEM_WRITE   = $0010;
+    PCI_COMMAND_VGA_PALETTE = $0020;
+    PCI_COMMAND_PARITY      = $0040;
+    PCI_COMMAND_WAIT        = $0080;
+    PCI_COMMAND_SERR        = $0100;
+    PCI_COMMAND_FAST_BACK   = $0200;
+    PCI_COMMAND_INT_DISABLE = $0400;
+    PCI_COMMAND_SERR_ENABLE = $8000;
+
+    PCI_CAP_ID_MSI           = $05;
+
+    // Bits in the MSI Control register (16-bit):
+    MSI_CONTROL_ENABLE       = 1 shl 0;  // Bit 0
+    MSI_CONTROL_64BIT        = 1 shl 7;  // Bit 7
+    MSI_CONTROL_PVMASK       = 1 shl 8;  // Bit 8 (optional: per-vector masking)
 type
 
     PPCI_Device = ^TPCI_Device;
@@ -44,6 +67,7 @@ type
         address1       : uint32;
         address2       : uint32;
         address3       : uint32;
+
         address4       : uint32;
         address5       : uint32;
         CIS_pointer    : uint32;

src/driver/storage/ATA_ISR.pas

@@ -1,85 +0,0 @@
-//  Copyright 2021 Aaron Hance
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-{ 
-	Drivers->Storage->ATA_ISR - Primary ATA IRQ.
-	
-	@author(Aaron Hance <[email protected]>)
-}
-unit ATA_ISR;
-
-interface
-
-uses
-    util,
-    syslog,
-    isr_types,
-    isrmanager,
-    IDT;
-
-procedure register();
-procedure hook(hook_method : uint32);
-procedure unhook(hook_method : uint32);
-
-implementation
-
-var
-    Hooks : Array[1..MAX_HOOKS] of pp_hook_method;
-
-procedure Main(); interrupt;
-var
-    i : integer;
-    
-begin
-    CLI;
-    for i:=0 to MAX_HOOKS-1 do begin
-        if uint32(Hooks[i]) <> 0 then Hooks[i](void($00000000));
-    end;
-    syslog.writestringln('Disk Operation Complete');
-end;
-
-procedure register();
-begin
-    memset(uint32(@Hooks[0]), 0, sizeof(pp_hook_method)*MAX_HOOKS);
-    //isrmanager.registerISR(76, @Main);
-    //IDT.set_gate(76, uint32(@Main), $08, ISR_RING_0);
-end;
-
-procedure hook(hook_method : uint32);
-var
-    i : uint32;
-
-begin
-    for i:=0 to MAX_HOOKS-1 do begin
-        if uint32(Hooks[i]) = hook_method then exit;
-    end;
-    for i:=0 to MAX_HOOKS-1 do begin
-        if uint32(Hooks[i]) = 0 then begin
-            Hooks[i]:= pp_hook_method(hook_method);
-            exit;
-        end;
-    end;
-end;
-
-procedure unhook(hook_method : uint32);
-var
-    i : uint32;
-begin
-    for i:=0 to MAX_HOOKS-1 do begin
-        If uint32(Hooks[i]) = hook_method then Hooks[i]:= nil;
-        exit;
-    end;
-end;
-
-end.

src/driver/storage/Filesystems/asfs.pas

@@ -22,14 +22,17 @@ unit asfs;
 interface
 
 uses
-    storagemanagement,
-    util,
-    lmemorymanager,
-    strings,
+    syslog,
     lists,
-    tracer,
+    lmemorymanager,
+    rtc,
     serial,
-    rtc;
+    storagemanagement,
+    storagemanager,
+    strings,
+    stdio,
+    tracer,
+    util;
 
 type
 
@@ -98,7 +101,7 @@ begin
     filesystemRecord^.endOfData       := filesystemRecord.sectorsPerTable; 
     //filesystemRecord.volumeLabel     := config^
 
-    disk^.writecallback(disk, start, 1, buffer);
+    storagemanager.storage_write(disk, start, 1, buffer);
     memset(uint32(buffer), 0, disk^.sectorSize);
 
     fileEntry            := PFileEntry(buffer)[0];
@@ -111,7 +114,7 @@ begin
     fileEntry^.attributes := 0;
     fileEntry^.location   := 1;
 
-    disk^.writecallback(disk, start + 1, 1, buffer);
+    storagemanager.storage_write(disk, start + 1, 1, buffer);
 
 
 end;

src/driver/storage/Filesystems/filesystemmanager.pas

@@ -0,0 +1,120 @@
+{
+    Driver->Storage->FilesystemManager - Filesystem driver registry and routing.
+
+    Filesystem drivers (FAT32, FlatFS, etc.) register themselves here.
+    When VolumeManager discovers a volume, FilesystemManager probes it
+    against all registered filesystem drivers to detect the filesystem type.
+
+    @author(Aaron Hance <[email protected]>)
+}
+unit filesystemmanager;
+
+interface
+
+uses
+    lists,
+    lmemorymanager,
+    storagetypes,
+    strings,
+    tracer,
+    util;
+
+var
+    filesystems : PLinkedListBase;
+
+procedure init();
+procedure register_filesystem(filesystem : PFilesystem);
+function get_filesystem_count() : uint32;
+function get_filesystem(index : uint32) : PFilesystem;
+function find_filesystem_by_name(name : pchar) : PFilesystem;
+function find_filesystem_by_id(system_id : uint8) : PFilesystem;
+procedure probe_volume(volume : PStorage_Volume);
+
+implementation
+
+procedure init();
+begin
+    push_trace('FilesystemManager.init');
+    filesystems := LL_New(sizeof(TFilesystem));
+end;
+
+procedure register_filesystem(filesystem : PFilesystem);
+var
+    elm : void;
+begin
+    push_trace('FilesystemManager.register_filesystem');
+
+    elm := LL_Add(filesystems);
+    memcpy(uint32(filesystem), uint32(elm), sizeof(TFilesystem));
+end;
+
+function get_filesystem_count() : uint32;
+begin
+    get_filesystem_count := LL_Size(filesystems);
+end;
+
+function get_filesystem(index : uint32) : PFilesystem;
+begin
+    if index < LL_Size(filesystems) then
+        get_filesystem := PFilesystem(LL_Get(filesystems, index))
+    else
+        get_filesystem := nil;
+end;
+
+function find_filesystem_by_name(name : pchar) : PFilesystem;
+var
+    i  : uint32;
+    fs : PFilesystem;
+begin
+    push_trace('FilesystemManager.find_filesystem_by_name');
+    find_filesystem_by_name := nil;
+
+    for i := 0 to LL_Size(filesystems) - 1 do begin
+        fs := PFilesystem(LL_Get(filesystems, i));
+        if stringEquals(fs^.sName, name) then begin
+            find_filesystem_by_name := fs;
+            exit;
+        end;
+    end;
+end;
+
+function find_filesystem_by_id(system_id : uint8) : PFilesystem;
+var
+    i  : uint32;
+    fs : PFilesystem;
+begin
+    push_trace('FilesystemManager.find_filesystem_by_id');
+    find_filesystem_by_id := nil;
+
+    for i := 0 to LL_Size(filesystems) - 1 do begin
+        fs := PFilesystem(LL_Get(filesystems, i));
+        if fs^.system_id = system_id then begin
+            find_filesystem_by_id := fs;
+            exit;
+        end;
+    end;
+end;
+
+procedure probe_volume(volume : PStorage_Volume);
+var
+    i  : uint32;
+    fs : PFilesystem;
+begin
+    push_trace('FilesystemManager.probe_volume.enter');
+
+    { Try each registered filesystem's identify callback against this volume }
+    for i := 0 to LL_Size(filesystems) - 1 do begin
+        fs := PFilesystem(LL_Get(filesystems, i));
+        if fs^.identifyCallback <> nil then begin
+            push_trace('FilesystemManager.probe_volume.tryFS');
+            if fs^.identifyCallback(volume) then begin
+                volume^.filesystem := fs;
+                push_trace('FilesystemManager.probe_volume.found');
+                exit;
+            end;
+        end;
+    end;
+    push_trace('FilesystemManager.probe_volume.exit');
+end;
+
+end.

src/driver/storage/IDE/idetypes.pas

@@ -0,0 +1,149 @@
+//  Copyright 2021 Aaron Hance
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+{ 
+	Driver->Storage->IDE->idetypes - IDE device types and constants.
+	
+	@author(Aaron Hance <[email protected]>)
+}
+unit idetypes;
+
+interface
+
+const
+
+    ATA_SR_BUSY = $80; //BUSY
+    ATA_SR_DRDY = $40; //DRIVE READY
+    ATA_SR_DF   = $20; //DRIVE WRITE FAULT
+    ATA_SR_DSC  = $10; //DRIVE SEEK COMPLETE
+    ATA_SR_DRQ  = $08; //DATA REQUEST READY
+    ATA_SR_CORR = $04; //CORRECTED DATA
+    ATA_SR_IDX  = $02; //INLEX
+    ATA_SR_ERR  = $01; //ERROR
+
+    ATA_ER_BBK   = $80; //BAD SECTOR
+    ATA_ER_UNC   = $40; //UNCORRECTABLE DATA
+    ATA_ER_MC    = $20; //NO MEDIA
+    ATA_ER_IDNF  = $10; //ID MARK NOT FOUND
+    ATA_ER_MCR   = $08; //NO MEDIA
+    ATA_ER_ABRT  = $04; //COMMAND ABORTED
+    ATA_ER_TK0NF = $02; //TRACK 0 NOT FOUND
+    ATA_ER_AMNF  = $01; //NO ADDRESS MARK
+
+    ATA_CMD_READ_PIO        = $20; 
+    ATA_CMD_READ_PIO_EXT    = $24; 
+    ATA_CMD_READ_DMA        = $C8; 
+    ATA_CMD_READ_DMA_EXT    = $25; 
+    ATA_CMD_WRITE_PIO       = $30; 
+    ATA_CMD_WRITE_PIO_EXT   = $34; 
+    ATA_CMD_WRITE_DMA       = $CA; 
+    ATA_CMD_WRITE_DMA_EXT   = $35; 
+    ATA_CMD_CACHE_FLUSH     = $E7; 
+    ATA_CMD_CACHE_FLUSH_EXT = $EA; 
+    ATA_CMD_PACKET          = $A0; 
+    ATA_CMD_IDENTIFY_PACKET = $A1; 
+    ATA_CMD_IDENTIFY        = $EC; 
+
+    ATAPI_CMD_READ  = $A8;
+    ATAPI_CMD_EJECT = $1B;
+
+    ATA_IDENT_DEVICETYPE   = $0;    
+    ATA_IDENT_CYLINDERS    = $2;    
+    ATA_IDENT_HEADS        = $6;    
+    ATA_IDENT_SECOTRS      = $12;    
+    ATA_IDENT_SERIAL       = $20;    
+    ATA_IDENT_MODEL        = $54;    
+    ATA_IDENT_CAPABILITIES = $98;    
+    ATA_IDENT_FIELDVALID   = $106;   
+    ATA_IDENT_MAX_LBA      = $120;   
+    ATA_IDENT_COMMANDSETS  = $164;       
+    ATA_IDENT_MAX_LBA_EXT  = $200;
+
+    ATA_REG_DATA       = $00;
+    ATA_REG_ERROR      = $01;
+    ATA_REG_FEATURES   = $01;
+    ATA_REG_SECCOUNT  = $02;
+    ATA_REG_LBA0       = $03;
+    ATA_REG_LBA1       = $04;
+    ATA_REG_LBA2       = $05;
+    ATA_REG_HDDEVSEL   = $06;
+    ATA_REG_COMMAND    = $07;
+    ATA_REG_STATUS     = $07;
+    ATA_REG_SECCOUNT1  = $08;
+    ATA_REG_LBA3       = $09;
+    ATA_REG_LBA4       = $0A;
+    ATA_REG_LBA5       = $0B;
+    ATA_REG_CONTROL    = $0C; //1FC on some systems, 3F6 on all
+    ATA_REG_ALTSTATUS  = $0C;
+    ATA_REG_DEVADDRESS = $0D; 
+
+    ATA_DEVICE_MASTER = $A0;
+    ATA_DEVICE_SLAVE = $B0;
+
+    ATA_PRIMARY_BASE = $1F0;
+    ATA_PRIMARY_BASE1 = $3F6;
+    ATA_SECONDARY_BASE = $170; //todo check this
+    ATA_SECONDARY_BASE1 = $376; //todo check this
+
+    ATA_INTERRUPT_PRIMARY = $3F6;
+    ATA_INTERRUPT_SECONDARY = $376;
+
+type
+    TPortMode = (P_READ, P_WRITE);
+
+    TIdentResponse = array[0..255] of uint16;
+    PIdentResponse = ^TIdentResponse;
+
+    TIDE_Channel_Registers = record
+        base    : uint16;
+        ctrl    : uint16;
+        bmide   : uint16;
+        noInter : uint8;
+    end;
+
+    TIDE_Status = bitpacked record 
+        Busy  : Boolean;
+        Ready : Boolean;
+        Fault : Boolean;
+        Seek  : Boolean;
+        DRQ   : Boolean;
+        CORR  : Boolean;
+        IDDEX : Boolean;
+        ERROR : Boolean;
+    end;
+    
+    PIDE_Status = ^TIDE_Status;
+
+    TIDE_Device = record
+        exists       : boolean;
+        isPrimary    : boolean;
+        isMaster     : boolean;
+        isATAPI      : boolean;
+        status       : TIDE_Status;
+        base         : uint16;
+        blockSize    : uint32;
+        info         : PIdentResponse;
+    end;
+
+    TIDE_PACKET = record
+        command : uint8;
+        lba     : uint32;
+        count   : uint8;
+        buffer  : puint16;
+    end;
+
+
+implementation
+  
+end.