Dynamic lists fixes and additions

.drone.yml

@@ -1,52 +0,0 @@
-kind: pipeline
-type: docker
-name: build
-
-steps:
-  - name: build-image
-    image: plugins/docker
-    settings:
-      repo: t3hn3rd/asuro-build
-      tags: latest
-      dockerfile: Dockerfile
-      registry: docker.io
-      username:
-        from_secret: docker_username
-      password:
-        from_secret: docker_password
-
-  - name: compile
-    image: t3hn3rd/asuro-build:latest
-    depends_on:
-      - build-image
-    commands:
-      - git fetch --tags
-      - find . -type f -print0 | xargs -0 dos2unix
-      - chmod +x /drone/src/*.sh
-      - /drone/src/compile.sh
-
-  - name: upload-iso-artifact
-    image: alpine/git
-    depends_on:
-      - compile
-    when:
-      branch:
-        - master
-    environment:
-      GITEA_TOKEN:
-        from_secret: gitea_token
-    commands:
-      - apk add --no-cache curl
-      - chmod +w Asuro.iso
-      - cp Asuro.iso "Asuro-$(git rev-parse --short HEAD).iso"
-      - echo "Uploading Asuro-$(git rev-parse --short HEAD).iso to Gitea Packages..."
-      - |
-         curl -X PUT "https://gitea.spexeah.com/api/packages/Spexeah/generic/asuro-iso/$(git rev-parse --short HEAD)/Asuro.iso" \
-         -H "Authorization: token $GITEA_TOKEN" --upload-file "Asuro-$(git rev-parse --short HEAD).iso"
-      - echo "Updating latest ISO reference..."
-      - |
-        curl -X DELETE https://gitea.spexeah.com/api/packages/Spexeah/generic/asuro-iso/latest/Asuro.iso \
-        -H "Authorization: token $GITEA_TOKEN" || echo "No previous latest version found."
-      - |
-        curl -X PUT https://gitea.spexeah.com/api/packages/Spexeah/generic/asuro-iso/latest/Asuro.iso \
-        -H "Authorization: token $GITEA_TOKEN" --upload-file "Asuro-$(git rev-parse --short HEAD).iso"

.gitignore

@@ -10,4 +10,5 @@
 /*.sh~
 /*.img
 src/include/asuro.pas
-localenv.json
+.vscode/launch.json
+.vscode

.gitlab-ci.yml

@@ -0,0 +1,73 @@
+stages:
+  - Compile Versions
+  - Compile Sources
+  - Link
+  - Generate Documentation
+  - Deploy
+
+cache:
+  - key: ${CI_COMMIT_REF_SLUG}
+    paths:
+      - lib/*.o
+      - bin/kernel.bin
+      - doc
+
+before_script:
+  - chmod +x *.sh
+
+versions:
+  stage: Compile Versions
+  script:
+    - ./compile_vergen.sh
+  artifacts:
+    paths:
+      - release/*.svg
+      - src/include/asuro.pas
+
+compile_sources:
+  stage: Compile Sources
+  script:
+    - rm -f lib/*.so
+    - ./compile_sources.sh
+  needs:
+    - versions
+
+link:
+  stage: Link
+  script:
+    - ./compile_stub.sh
+    - ./compile_link.sh
+  needs:
+    - versions
+    - compile_sources
+
+isogen:
+  stage: Deploy
+  script:
+    - ./compile_isogen.sh
+    - ./compile_sumgen.sh
+  artifacts:
+    paths:
+      - ./Asuro.iso
+      - ./release/checksum.svg
+  needs:
+    - link
+
+docgen:
+  stage: Generate Documentation
+  only:
+    - master
+    - develop
+  script:
+    - ./compile_sourcelist.sh
+    - ./compile_docs.sh
+    # Remove comments when we want to use gitlab pages.
+    #- cp doc public
+  allow_failure: true
+  artifacts:
+    paths:
+      - doc
+      #- public/*
+      #- ./sources.list
+  needs:
+    - versions

.vscode/launch.json

@@ -3,11 +3,18 @@
         {
             "name":"Run",
             "request": "launch",
-            "type": "PowerShell",
+            "type": "coreclr",
             "preLaunchTask": "Build",
-            "script": "${workspaceFolder}/virtualbox-wrapper.ps1",
-            "args": ["-Command", "up"],
+            "program": "VBoxSDL",
+            "args": [
+                "--comment", 
+                "Asuro", 
+                "--startvm", 
+                "7d395c96-891c-4139-b77d-9b6b144b0b93"
+            ],
             "cwd": "${workspaceFolder}",
+            "console": "internalConsole",
+            "internalConsoleOptions": "neverOpen"
         }
     ]
 }

.vscode/tasks.json

@@ -9,18 +9,14 @@
             "command": "docker-compose",
             "args": [
                 "run",
-                "builder",
+                "builder"
             ],
             "type": "shell",
             "problemMatcher": [],
             "group": {
                 "kind": "build",
                 "isDefault": true
-            },
-            "dependsOn": [
-                "Close VirtualBox",
-                "Clean"
-            ]
+            }
         },
         {
             "label": "Build (Builder)",
@@ -30,24 +26,6 @@
                 "builder"
             ],
             "type": "shell"
-        },
-        {
-            "label": "Clean",
-            "command": "docker-compose",
-            "args": [
-                "down",
-                "--remove-orphans"
-            ],
-            "type": "shell"
-        },
-        {
-            "label": "Close VirtualBox",
-            "command": "./virtualbox-wrapper.ps1",
-            "args": [
-                "-Command", 
-                "down"
-            ],
-            "type": "shell"
         }
     ]
 }

Dockerfile

@@ -16,7 +16,7 @@ RUN curl -sL https://sourceforge.net/projects/freepascal/files/Linux/$FPC_VERSIO
 
 COPY compile.sh /compile.sh
 ADD https://raw.githubusercontent.com/fsaintjacques/semver-tool/master/src/semver /usr/bin/semver
-RUN chmod +x /usr/bin/semver
+RUN mkdir /code && chmod +x /usr/bin/semver
 WORKDIR /code
 RUN find . -type f -print0 | xargs -0 dos2unix
 ENTRYPOINT ["/bin/bash", "-c"]

compile.sh

@@ -20,30 +20,26 @@ runOrFail() {
 	fi
 }
 
-declare -a run_steps=(
-	"compile_stub.sh" "Failed to compile stub!"
-	"compile_vergen.sh" "Versions failed to compile"
-	"compile_sources.sh" "Failed to compile FPC Sources!"
-	"compile_link.sh" "Failed linking!"
-	"compile_isogen.sh" "Failed to create ISO!"
-)
+runOrFail $(pwd)/compile_stub.sh "Failed to compile stub!"
 
-for ((i=0; i<${#run_steps[@]}; i+=2))
-do
-	if [ "$ERRCOUNT" -eq "0" ]
-	then
-			script=$(pwd)/"${run_steps[$i]}"
-			message="${run_steps[$i+1]}"
-			runOrFail "$script" "$message"
-	fi
-done
+#Generate .pas with versioning headers.
+runOrFail $(pwd)/compile_vergen.sh "Versions failed to compile"
+
+#Compile all .pas sources
+runOrFail $(pwd)/compile_sources.sh "Failed to compile FPC Sources!"
+
+#Link into a binary.
+runOrFail $(pwd)/compile_link.sh "Failed linking!"
+
+#Generate an ISO with GRUB as the Bootloader.
+runOrFail ./compile_isogen.sh "Failed to create ISO!"
 
 #Call generate final artifacts based on failure or success of the above.
 if [ "$ERRCOUNT" -ne "0" ]
 then
-	. ./compile_finish.sh "failed"
+	./compile_finish.sh "failed"
 else
-	. ./compile_finish.sh "success"
+	./compile_finish.sh "success"
 fi
 
 cd ..

compile_sources.sh

@@ -4,4 +4,4 @@ echo "======================="
 echo " "
 echo "Compiling FPC Sources..."
 echo " "
-fpc -Aelf -gw -g -gl -n -vlewn -O3 -Op3 -Si -Sc -Sg -Xd -CX -XXs -CfSSE -CfSSE2 -Rintel -Pi386 -Tlinux -FElib/ -Fusrc/* -Fusrc/driver/* -Fusrc/driver/net/* src/kernel.pas
+fpc -Aelf -gw -n -va -O3 -Op3 -Si -Sc -Sg -Xd -CX -XXs -CfSSE -CfSSE2 -Rintel -Pi386 -Tlinux -FElib/ -Fusrc/* -Fusrc/driver/* -Fusrc/driver/net/* src/kernel.pas

docker-compose.yml

@@ -1,3 +1,4 @@
+version: "3.9"
 services:
   builder:
     build: .

linker.script

@@ -2,7 +2,7 @@ ENTRY(loader)
 SECTIONS
 {
   . = 0xC0100000;
-  kernel_start = .;
+
   .text : AT(ADDR(.text) - 0xC0000000)
   {
     text = .; _text = .; __text = .;
@@ -33,5 +33,4 @@ SECTIONS
     . = ALIGN(4096);
   }
   end = .; _end = .; __end = .;
-  kernel_end = .;
 }

readme.md

@@ -13,8 +13,8 @@ We welcome everyone to give building/breaking/fixing/shooting Asuro a go, feel f
     I don't think this needs an explaination.
 * [VSCode (Optional, but highly recommended)](https://code.visualstudio.com/)
     Visual Studio code is our IDE of choice, and we have a number of recommended plugins.
-    * [PowerShell Plugin by Microsoft](https://marketplace.visualstudio.com/items?itemName=ms-vscode.powershell)
-        This plugin gives you the ability to use the 'PowerShell' task type, allowing the automatic launching of virtualbox with the resulting image generated during compilation of Asuro.
+    * [C# Plugin by Microsoft](https://marketplace.visualstudio.com/items?itemName=ms-dotnettools.csharp)
+        This plugin gives you the ability to use the 'coreclr' task type, allowing the automatic launching of virtualbox with the resulting image generated during compilation of Asuro.
 * [VirtualBox](https://www.virtualbox.org/)
     Virtualbox is our Virtualisation environment of choice, don't ask why, it just is.
 
@@ -27,7 +27,7 @@ We welcome everyone to give building/breaking/fixing/shooting Asuro a go, feel f
 3. Install Docker for Windows.
 4. Install Git for Windows.
 5. Install VSCode & the listed plugins.
-6. Install VirtualBox (v7+).
+6. Install VirtualBox.
 7. Clone this repository.
 8. Run the following command in the root of the repo to build the docker image:
     ```powershell
@@ -50,29 +50,32 @@ We welcome everyone to give building/breaking/fixing/shooting Asuro a go, feel f
     ```xml
     <Machine uuid="{7d395c96-891c-4139-b77d-9b6b144b0b93}" name="Asuro" OSType="Linux" snapshotFolder="Snapshots" lastStateChange="2021-06-20T20:33:07Z">
     ```
-    Copy the uuid, in our case `7d395c96-891c-4139-b77d-9b6b144b0b93` & create a `localenv.json` file in the project root with the following content:
+    Copy the uuid, in our case `7d395c96-891c-4139-b77d-9b6b144b0b93` and replace the uuid found in `.vscode\launch.json` under `args`, so that it looks something like this:
     ```json
     {
-        "VirtualBox":{
-            "MachineName":"<YOUR_UUID_OR_MACHINE_NAME>"
-        }
+        "configurations": [
+            {
+                "name":"Run",
+                "request": "launch",
+                "type": "coreclr",
+                "preLaunchTask": "Build",
+                "program": "VBoxSDL",
+                "args": [
+                    "--comment", 
+                    "Asuro", 
+                    "--startvm", 
+                    "<YOUR UUID HERE>"
+                ],
+                "cwd": "${workspaceFolder}",
+                "console": "internalConsole",
+                "internalConsoleOptions": "neverOpen"
+            }
+        ]
     }
     ```
     This will allow VSCode to automatically launch VirtualBox once Asuro has been compiled.
-    
-    You can also enable the serial adapter "COM1" in mode "Raw File", give it a path, and provide this path in the `localenv.json` as follows:
-    ```json
-    {
-        "VirtualBox" : {
-            "MachineName": "<YOUR_UUID_OR_MACHINE_NAME>",
-            "LogLocation": "Fully\\Qualified\\Path\\To\\Your\\Log\\File"
-        }
-    }
-    ```
-    This will allow you to see the console output from Asuro in your host terminal.
 13. Open your project folder in VSCode, use CTRL+SHIFT+B to build & F5 to build + run in VBox.
 14. Congratulations! You can now play with Asuro!
 
-### Notes & Gotchas
-- The above process has been updated to be compatible with VirtualBox 7+, in which VBoxSDL was removed and vboxmanage should be used in its place. A small wrapper powershell script is used to achieve this.
+### Gotchas
 - It was noted that Windows builds above `20H2` seem to have issues installing WSL2. We may have to wait for a patch from Microsoft to fix this. Our devs are currently using build `20H2`.

src/include/lists.pas

@@ -44,6 +44,19 @@ type
         ElementSize : uint32;
     end;
 
+    { Dynamic List }
+
+    PDList = ^TDList;
+    TDList = record
+        Count : uint32;
+        Data : void;
+        ElementSize : uint32;
+        DataSize : uint32;
+
+    end;
+    
+    { Dynamic List Iterator } 
+
 { String Linked List }
 
 procedure  STRLL_Add(LinkedList : PLinkedListBase; str : pchar);
@@ -66,6 +79,24 @@ function  LL_Get(LinkedList : PLinkedListBase; idx : uint32) : Void;
 procedure LL_Free(LinkedList : PLinkedListBase);
 function  LL_FromString(str : pchar; delimter : char) : PLinkedListBase;
 
+    
+{ Dynamic List }
+
+function  DL_New(ElementSize : uint32) : PDList;
+function  DL_Add(DList : PDList) : Void;
+function  DL_Delete(DList : PDList; idx : uint32) : boolean;
+function  DL_Size(DList : PDList) : uint32;
+function  DL_Set(DList : PDList; idx : uint32; elm : puint32) : boolean;
+function  DL_Get(DList : PDList; idx : uint32) : Void;
+procedure DL_Free(DList : PDList);
+function  DL_IndexOf(DList : PDList; elm : puint32) : uint32;
+function  DL_Contains(DList : PDList; elm : puint32) : boolean;
+function DL_Concat(DList1 : PDList; DList2 : PDList) : PDList;
+
+
+{ Dynamic List Iterator }
+
+
 implementation
 
 uses
@@ -351,4 +382,227 @@ begin
     end;
 end;
 
+{ Dynamic List }
+
+function  DL_New(ElementSize : uint32) : PDList;
+var 
+    DL : PDList;
+begin
+
+    DL := PDList(kalloc(sizeof(DL)));
+    DL^.ElementSize:= ElementSize;
+    DL^.count := 0;
+
+    if ElementSize > 128 then begin
+        DL^.data := kalloc(ElementSize * 4);
+        Dl^.DataSize:= ElementSize * 4;
+
+    end else if ElementSize > 64 then begin
+        DL^.data := kalloc(ElementSize * 8);
+        Dl^.DataSize:= ElementSize * 8;
+        
+    end else if ElementSize > 32 then begin
+        DL^.data := kalloc(ElementSize * 16);
+        Dl^.DataSize:= ElementSize * 16;
+
+    end else if ElementSize > 16 then begin
+      DL^.data := kalloc(ElementSize * 32);
+      Dl^.DataSize:= ElementSize * 32;
+    end else begin
+      DL^.data := kalloc(ElementSize * 64);
+      Dl^.DataSize:= ElementSize * 64;
+    end;
+
+    DL_New := DL;
+end;
+
+function  DL_Add(DList : PDList) : Void;
+var 
+    elm : puint32;
+    tempList : puint32;
+begin
+
+    //check if we need to resize
+    if (DList^.count + 1) * DList^.ElementSize > DList^.DataSize then begin
+        push_trace('lists.DL_Add: Resizing');
+        tempList := DList^.Data;
+        DList^.Data := kalloc(DList^.DataSize * 2);
+        
+        memset(uint32(DList^.Data), 0, DList^.DataSize * 2);
+        memcpy(uint32(tempList), uint32(DList^.Data), DList^.DataSize);
+
+        DList^.DataSize:= DList^.DataSize * 2;
+
+        kfree(void(tempList));
+    end;
+
+    push_trace('lists.DL_Add');
+
+
+    elm := puint32(@puint8(DList^.Data)[(DList^.count * DList^.ElementSize)]);
+    elm^ := 0;
+
+    push_trace('lists.DL_Add: Adding');
+
+    DList^.count := DList^.count + 1;
+
+    DL_Add := elm;
+end;
+
+function  DL_Delete(DList : PDList; idx : uint32) : boolean;
+var 
+    elm : puint32;
+    tempList : puint32;
+begin
+    if idx >= DList^.count then begin
+        DL_Delete := false;
+        exit;
+    end;
+
+    elm := puint32( puint8(DList^.Data) + (idx * DList^.ElementSize));
+    memcpy(uint32( puint8(elm) + DList^.ElementSize), uInt32(elm), (DList^.count - idx) * DList^.ElementSize);
+
+    DList^.count := DList^.count - 1;
+
+    if (DList^.count * DList^.ElementSize) < DList^.DataSize DIV 2 then begin
+        tempList := DList^.Data;
+        DList^.DataSize:= DList^.DataSize DIV 2;
+
+        DList^.Data := kalloc(DList^.DataSize);
+        memcpy(uint32(tempList), uint32(DList^.Data), DList^.DataSize);
+
+        kfree(void(tempList));
+    end;
+
+    DL_Delete := true;
+end;
+
+function  DL_Get(DList : PDList; idx : uint32) : Void;
+var 
+    elm : puint32;
+begin
+    if idx >= DList^.count then begin
+        DL_Get := nil;
+        exit;
+    end;
+
+    elm := puint32(DList^.Data + (idx * DList^.ElementSize));
+    DL_Get := elm;
+end;
+
+function DL_Set(DList : PDList; idx : uint32; elm : puint32) : boolean;
+var
+    tempList : PuInt32;
+    size : uint32;
+begin
+    if idx >= DList^.count then begin
+        DL_Set := false;
+        exit;
+    end;
+
+    //check if we need to resize
+    if (idx + 1) * Dlist^.ElementSize > DList^.DataSize then begin
+                console.writestring('resising thisthinghere');
+
+        size := idx * DList^.ElementSize * 2;
+
+        tempList := DList^.Data;
+        DList^.Data := kalloc(size);
+        memset(uint32(DList^.Data), 0, size);
+
+        memcpy(uint32(tempList), uint32(DList^.Data), DList^.DataSize);
+
+        DList^.DataSize := size;
+
+        kfree(void(tempList));
+    end;
+
+    console.writeString('offset: ');
+    console.writeintln(idx * DList^.ElementSize);
+    memcpy(uint32(elm), uint32( PuInt8(DList^.Data) + (idx * DList^.ElementSize)), DList^.ElementSize);
+
+    //check if count is smaller than idx and if so, increase count
+    if DList^.count < idx then DList^.count := idx;
+
+    DL_Set := true;
+end;
+
+function DL_Size(DList : PDList) : uint32;
+begin
+    DL_Size := DList^.count;
+end;
+
+procedure DL_Free(DList : PDList);
+begin
+    kfree(void(DList^.Data));
+    kfree(void(DList));
+end;
+
+function DL_IndexOf(DList : PDList; elm : puint32) : uint32;
+var 
+    i : uint32;
+    temp : puint32;
+begin
+    for i := 0 to DList^.count - 1 do begin
+        temp := puint32( puint8(DList^.Data) + (i * DList^.ElementSize));
+        if temp = elm then begin
+            DL_IndexOf := i;
+            exit;
+        end;
+    end;
+
+    DL_IndexOf := -1;
+end;
+
+function DL_Contains(DList : PDList; elm : puint32) : boolean;
+var 
+    i : uint32;
+begin
+    i := DL_IndexOf(DList, elm);
+
+    if i = -1 then begin
+        DL_Contains := false;
+        exit;
+    end;
+
+    DL_Contains := true;
+end;
+
+function DL_Concat(DList1 : PDList; DList2 : PDList) : PDList;
+var 
+    i : uint32;
+    temp : puint32;
+begin
+
+    //check element size
+    if DList1^.ElementSize <> DList2^.ElementSize then begin
+        DL_Concat := nil;
+        exit;
+    end;
+
+    //check if we need to resize
+    while true do begin
+          if (DList1^.count + DList2^.count) * DList1^.ElementSize > DList1^.DataSize then begin
+
+            temp := DList1^.Data;
+            DList1^.Data := kalloc(DList1^.DataSize * 2);
+            
+            memset(uint32(DList1^.Data), 0, DList1^.DataSize * 2);
+            memcpy(uint32(temp), uint32(DList1^.Data), DList1^.DataSize);
+
+            DList1^.DataSize:= DList1^.DataSize * 2;
+
+            kfree(void(temp));
+        end else Break;
+    end;
+
+
+    for i := 0 to DList2^.count - 1 do begin
+        temp := DL_Add(DList1);
+        memcpy(uint32(DL_Get(DList2, i)), uint32(temp), DList1^.ElementSize);
+    end;
+
+    DL_Concat := DList1;
+end;
+
 end.

src/include/system.pas

@@ -143,22 +143,6 @@ type
 
     PText = ^Text;
 
-var
-  AK_START : uint32; external name 'kernel_start';
-  AK_END   : uint32; external name 'kernel_end';
-  ASURO_KERNEL_START : uint32;
-  ASURO_KERNEL_END   : uint32;
-  ASURO_KERNEL_SIZE  : uint32;
-
-procedure init();
-
 implementation
 
-procedure init();
-begin
-    ASURO_KERNEL_START := uint32(@AK_START);
-    ASURO_KERNEL_END := uint32(@AK_END);
-    ASURO_KERNEL_SIZE:= ASURO_KERNEL_END - ASURO_KERNEL_START;
-end;
-
 end.

src/kernel.pas

@@ -125,9 +125,6 @@ var
    HM              : PHashMap;
    
 begin
-     { Init the base system unit }
-     System.init();
-
      { Serial Init }
      serial.init();
 

virtualbox-wrapper.ps1

@@ -1,58 +0,0 @@
-<# 
-You need a local git-ignored localenv.json file with the following content:
-{
-  "virtualbox": {
-    "MachineName": "your-machine-name or guid"
-  }
-}
-#>
-
-param (
-  [Parameter(Mandatory=$true)]
-  [ValidateSet('up', 'down')]
-  [String]$Command
-)
-
-$Config = Get-Content .\localenv.json | ConvertFrom-Json
-$MachineName = $Config.virtualbox.MachineName
-$LogLocation = $Config.virtualbox.LogLocation
-$LogOutputEnabled = $LogLocation -ne $null
-
-if ($Command -eq 'up') {
-
-  if($LogOutputEnabled) {
-    Clear-Content $LogLocation
-  }
-
-  $MonitorJob = Start-Job -ArgumentList $MachineName -ScriptBlock {
-    param($MachineName)
-    Write-Output "Starting $MachineName"
-    VBoxManage.exe startvm $MachineName
-    $running=$true
-    while($running) {
-      $status=(VBoxManage.exe list runningvms)
-      if($status) {
-        $running=$status.contains($MachineName)
-      } else {
-        $running=$false
-      }
-    }
-  }
-  
-  if($LogOutputEnabled) {
-    $LogJob = Start-Job -ArgumentList $LogLocation -ScriptBlock {
-      param($LogLocation)
-      Get-Content -Path $LogLocation -Wait
-    }
-  }
-
-  while($MonitorJob.State -eq 'Running') {
-    if($LogOutputEnabled) {
-      Receive-Job $LogJob
-    }
-    Receive-Job $MonitorJob
-  }
-} elseif ($Command -eq 'down') {
-  Write-Output "Stopping $MachineName"
-  VBoxManage.exe controlvm $MachineName poweroff
-}