From dd3bb1e016fe2241112b97999e1a1fe59cdb7fbb Mon Sep 17 00:00:00 2001 From: t3hn3rd Date: Sat, 7 Nov 2026 21:49:06 +0000 Subject: [PATCH] Automate boot unit inclusion with architecture-aware Moderne analysis - Replace boot.mgr's static unit list with generated dangling_units.inc - Generate Pascal LSTs and run the dangling-unit recipe before compilation - Exclude Wasuro, inactive architectures, and NOT_DANGLING units - Add ASURO_ARCH selection with an x86 default - Mark unfinished legacy units as NOT_DANGLING - Inherit pinned Moderne tooling in the builder image - Ensure Drone pulls the latest builder image - Document the updated build pipeline in MkDocs Validation: - Full x86 ISO build and VirtualBox boot succeeded - 1,133 kernel tests passed with no failures - x86_64 build failed clearly due to unavailable sources --- .drone.yml | 1 + .gitignore | 6 +- Dockerfile | 8 +- Dockerfile.dockerignore | 8 ++ doc/toolchain/dangling_units.md | 45 ++++++++ docker-compose.yml | 5 +- mkdocs.yml | 1 + readme.md | 3 +- src/arch/x86/isr/arch.x86.isr.tmr1.pas | 2 + src/arch/x86/proc/arch.x86.proc.loader.pas | 4 +- src/boot/boot.mgr.pas | 104 +----------------- .../ctl/ide/driver.storage.ctl.ide.ata.pas | 4 +- .../ctl/ide/driver.storage.ctl.ide.atapi.pas | 4 +- .../ctl/ide/driver.storage.ctl.ide.pas | 2 + .../storage/fs/driver.storage.fs.asfs.pas | 6 +- .../storage/vol/driver.storage.vol.table.pas | 2 + toolchain/architecture.sh | 12 ++ toolchain/compile.sh | 7 +- toolchain/compile_dangling.sh | 26 +++++ toolchain/compile_sources.sh | 20 +++- 20 files changed, 149 insertions(+), 121 deletions(-) create mode 100644 Dockerfile.dockerignore create mode 100644 doc/toolchain/dangling_units.md create mode 100644 toolchain/architecture.sh create mode 100644 toolchain/compile_dangling.sh diff --git a/.drone.yml b/.drone.yml index 2d6d7ef7..e53ba30f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -17,6 +17,7 @@ steps: - name: compile image: t3hn3rd/asuro-build:latest + pull: always depends_on: - build-image commands: diff --git a/.gitignore b/.gitignore index ccf7807c..58995960 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,8 @@ lessons_learnt.md wasuro src/core/core.version.pas site/* -!site/.gitkeep \ No newline at end of file +!site/.gitkeep +/src/boot/dangling_units.inc +/lib/dangling/ +.moderne +CLAUDE.md diff --git a/Dockerfile b/Dockerfile index 66febdd2..47d9d3c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM ubuntu:latest +ARG RECIPES_IMAGE=t3hn3rd/moderne-asuro-recipes:0.1.1@sha256:eaf54188053511dad9c620a8fb98b0f50c41770cd49a0b3d71a0538386522011 +FROM ${RECIPES_IMAGE} VOLUME ["/code"] @@ -7,7 +8,7 @@ RUN dpkg --add-architecture i386 RUN apt-get update && apt-get install -y \ curl dos2unix wget git make nasm binutils xorriso grub-pc-bin gcc gcc-multilib \ python3 python3-pip python3-pil && \ - apt-get clean my room + apt-get clean && rm -rf /var/lib/apt/lists/* RUN pip3 install --no-cache-dir --break-system-packages "mkdocs>=1.6,<2" mkdocs-material @@ -20,6 +21,5 @@ RUN curl -sL https://sourceforge.net/projects/freepascal/files/Linux/$FPC_VERSIO ADD https://raw.githubusercontent.com/fsaintjacques/semver-tool/master/src/semver /usr/bin/semver RUN chmod +x /usr/bin/semver WORKDIR /code -RUN find . -type f -print0 | xargs -0 dos2unix ENTRYPOINT ["/bin/bash", "-c"] -CMD ["find toolchain -name '*.sh' -exec dos2unix {} + 2>/dev/null; find toolchain -name '*.sh' -exec chmod +x {} +; bash toolchain/compile.sh"] \ No newline at end of file +CMD ["find toolchain -name '*.sh' -exec dos2unix {} + 2>/dev/null; find toolchain -name '*.sh' -exec chmod +x {} +; bash toolchain/compile.sh"] diff --git a/Dockerfile.dockerignore b/Dockerfile.dockerignore new file mode 100644 index 00000000..67377fd2 --- /dev/null +++ b/Dockerfile.dockerignore @@ -0,0 +1,8 @@ +**/.git +**/.gradle +**/build +**/.moderne +wasuro +site +lib +release diff --git a/doc/toolchain/dangling_units.md b/doc/toolchain/dangling_units.md new file mode 100644 index 00000000..bd64d7ac --- /dev/null +++ b/doc/toolchain/dangling_units.md @@ -0,0 +1,45 @@ +# Automatic boot-unit inclusion + +Run the usual `docker compose build builder` and `docker compose run builder` +from Asuro. Only this repository is required. Compose and Drone use the same +Dockerfile and repository build context, producing `t3hn3rd/asuro-build:latest`. +The Dockerfile inherits the published `t3hn3rd/moderne-asuro-recipes:0.1.1` image, +pinned by digest. It already includes Java, Moderne CLI, the Pascal LST generator, +and the registered recipe. The Asuro layer installs FPC and the kernel build tools. + +To upgrade tooling, publish the upstream images first and update `RECIPES_IMAGE` +in the Dockerfile to the new tag and digest. Local tooling image testing can use +`docker compose build --build-arg RECIPES_IMAGE= builder`. + +`ASURO_ARCH` selects the source architecture, defaulting to `x86`. For example: + +```sh +export ASURO_ARCH=x86 +docker compose run --rm builder +``` + +Compose forwards this variable to the builder; Drone inherits it from its compile +environment. Only `src/arch/$ASURO_ARCH` is added to FPC's unit search paths. +Other architecture folders are excluded from LST generation and from the recipe's +dependency graph and results. Common units outside `src/arch` remain eligible. +The selected architecture folder must exist. This source selection does not +change the existing i386 compiler target, assembly stub, or linker settings; +an x86_64 port still needs those platform-specific changes. + +`compile_dangling.sh` runs after Wasuro synchronization and before FPC. It copies +only `src/` and `compat/` into an isolated temporary analysis repository, generates +LSTs with rewrite-pascal, runs the standalone Moderne recipe, exports its +datatable, and writes `src/boot/dangling_units.inc`. Wasuro is not parsed. +Datatable conversion is provided by the standalone project's Java exporter. +Parse errors, recipe failures, and malformed datatables stop the build. The +generated include is ignored by Git and removed before each generation attempt. + +Use `{$DEFINE NOT_DANGLING}` near a unit declaration to exclude an experimental +or legacy unit from automatic inclusion. It can still be referenced normally. +This is a file-level marker; conditional definitions are not evaluated. The +analysis follows both interface and implementation uses, without expanding +includes or evaluating conditional compilation. + +Moderne runs through its CLI normally; any license or tenant configuration +required by your CLI installation must be provided to the container separately. +No host credentials are copied into the image. diff --git a/docker-compose.yml b/docker-compose.yml index 845189bd..7ef27af3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,8 @@ services: builder: + image: t3hn3rd/asuro-build:latest build: . + environment: + ASURO_ARCH: ${ASURO_ARCH:-x86} volumes: - - .:/code \ No newline at end of file + - .:/code diff --git a/mkdocs.yml b/mkdocs.yml index cba7f458..295faeed 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -283,6 +283,7 @@ nav: - Version Generation: toolchain/compile_vergen.md - LVGL Compilation: toolchain/compile_lvgl.md - Wasuro Pull: toolchain/compile_wasuro.md + - Automatic Boot Units: toolchain/dangling_units.md - Source List: toolchain/compile_sourcelist.md - Source Compilation: toolchain/compile_sources.md - Checksum: toolchain/compile_checksum.md diff --git a/readme.md b/readme.md index 1ab8b627..2027e0cc 100644 --- a/readme.md +++ b/readme.md @@ -28,7 +28,8 @@ We welcome everyone to give building/breaking/fixing/shooting Asuro a go, feel f 4. Install Git for Windows. 5. Install VSCode & the listed plugins. 6. Install VirtualBox (v7+). -7. Clone this repository. +7. Clone this repository. The builder inherits the published Moderne/Pascal + tooling image; see [boot-unit generation](doc/toolchain/dangling_units.md). 8. Run the following command in the root of the repo to build the docker image: ```powershell docker compose build builder diff --git a/src/arch/x86/isr/arch.x86.isr.tmr1.pas b/src/arch/x86/isr/arch.x86.isr.tmr1.pas index df373b7a..c347d473 100644 --- a/src/arch/x86/isr/arch.x86.isr.tmr1.pas +++ b/src/arch/x86/isr/arch.x86.isr.tmr1.pas @@ -19,6 +19,8 @@ } unit arch.x86.isr.tmr1; +{$DEFINE NOT_DANGLING} { Unused legacy timer ISR; not part of the kernel boot sequence. } + interface uses diff --git a/src/arch/x86/proc/arch.x86.proc.loader.pas b/src/arch/x86/proc/arch.x86.proc.loader.pas index 5c42b8d6..128cb61f 100644 --- a/src/arch/x86/proc/arch.x86.proc.loader.pas +++ b/src/arch/x86/proc/arch.x86.proc.loader.pas @@ -19,8 +19,10 @@ } unit arch.x86.proc.loader; +{$DEFINE NOT_DANGLING} { Unimplemented process-loader stub. } + interface implementation; -end. \ No newline at end of file +end. diff --git a/src/boot/boot.mgr.pas b/src/boot/boot.mgr.pas index 64ab9179..745f0499 100644 --- a/src/boot/boot.mgr.pas +++ b/src/boot/boot.mgr.pas @@ -71,107 +71,9 @@ procedure run; implementation uses - io.syslog, boot.splash, - - // Silo units that must be included here to compile - driver.storage.boot, - test, - core, - app.wasm, - driver.storage.ctl.ahci, - core.enc.base64, - core.panic, - driver.video.bga, - arch.x86.bda, - arch.x86.panic, - core.ds.cfifo, - core.ds.cfifols, - core.ds.circ, - core.gfx.color, - arch.x86.proc.sched, - arch.x86.cpu, - driver.video.desktop, - driver.video.doublebuffer, - driver.mgr, - driver.net.dev.e1000, - driver.bus.usb.ehci, - driver.storage.fs.fat32, - arch.x86.fault, - core.ds.fifo, - driver.storage.fs.mgr, - driver.storage.fs.flatfs, - core.gfx.fonts, - arch.x86.gdt, - driver.video.gpu, - svc.gfxd, - core.ds.hashmap, - arch.x86.idt, - arch.x86.isr.ioapic, - arch.x86.irq, - driver.storage.fs.iso9660, - arch.x86.isr, - arch.x86.isr.mgr, - driver.hid.keyboard, - core.ds.lifo, - core.ds.lists, - memory.heap, - driver.video.lvgl, - core.ds.maxh, - core.enc.md5, - core.ds.minh, - driver.hid.mouse, - arch.x86.multiboot, - driver.net, - driver.bus.usb.ohci, - app.partcmd, - driver.bus.pci, - arch.x86.memory.physical, - core.ds.prio, - proc.mgr, - app.mgr, - driver.hid.ps2.keyboard, - driver.hid.ps2.mouse, - core.rand, - driver.timer.rtc, - driver.io.serial, - io.stdio, - driver.storage.mgr, - driver.storage.test, - core.strings, - driver.exp.testdriver, - proc.testprocs, - arch.x86.isr.tmr0, - debug.tracer, - driver.bus.usb.uhci, - app.uidebug, - driver.bus.usb, - driver.hid.usb.keyboard, - driver.hid.usb.mouse, - driver.storage.ctl.usb, - driver.bus.usb.core, - svc.usbd, - driver.bus.usb.hub, - driver.bus.usb.types, - core.util, - arch.x86.util, - arch.x86.v86, - driver.video.vesa, - driver.storage.vfs, - driver.video, - arch.x86.memory.virtual, - core.enc.fnv1a, - core.enc.djb2, - core.ds.bloom, - app.volcmd, - driver.storage.vol.mgr, - app.vterminal, - wasm, - wasm.vm.io, - wasm.test, - wasm.test.framework, - driver.video.windows, - driver.bus.usb.xhci, - core.fmt.json; + io.syslog, boot.splash + {$INCLUDE dangling_units.inc} + ; type { Resolved execution order — indices into Entries[] } diff --git a/src/driver/storage/ctl/ide/driver.storage.ctl.ide.ata.pas b/src/driver/storage/ctl/ide/driver.storage.ctl.ide.ata.pas index 1ec18ad2..53256137 100644 --- a/src/driver/storage/ctl/ide/driver.storage.ctl.ide.ata.pas +++ b/src/driver/storage/ctl/ide/driver.storage.ctl.ide.ata.pas @@ -19,6 +19,8 @@ } unit driver.storage.ctl.ide.ata; +{$DEFINE NOT_DANGLING} { Part of the unfinished legacy IDE stack. } + interface uses @@ -248,4 +250,4 @@ begin pop_trace(); end; -end. \ No newline at end of file +end. diff --git a/src/driver/storage/ctl/ide/driver.storage.ctl.ide.atapi.pas b/src/driver/storage/ctl/ide/driver.storage.ctl.ide.atapi.pas index bee3ff92..6790bfe7 100644 --- a/src/driver/storage/ctl/ide/driver.storage.ctl.ide.atapi.pas +++ b/src/driver/storage/ctl/ide/driver.storage.ctl.ide.atapi.pas @@ -24,6 +24,8 @@ unit driver.storage.ctl.ide.atapi; +{$DEFINE NOT_DANGLING} { Nonfunctional legacy ATAPI driver. } + interface uses @@ -445,4 +447,4 @@ end; -end. \ No newline at end of file +end. diff --git a/src/driver/storage/ctl/ide/driver.storage.ctl.ide.pas b/src/driver/storage/ctl/ide/driver.storage.ctl.ide.pas index 72ee7ea2..3456a739 100644 --- a/src/driver/storage/ctl/ide/driver.storage.ctl.ide.pas +++ b/src/driver/storage/ctl/ide/driver.storage.ctl.ide.pas @@ -21,6 +21,8 @@ } unit driver.storage.ctl.ide; +{$DEFINE NOT_DANGLING} { Unfinished legacy IDE stack; requires removed console/terminal units. } + interface uses diff --git a/src/driver/storage/fs/driver.storage.fs.asfs.pas b/src/driver/storage/fs/driver.storage.fs.asfs.pas index 8edc64e9..4c36f8e2 100644 --- a/src/driver/storage/fs/driver.storage.fs.asfs.pas +++ b/src/driver/storage/fs/driver.storage.fs.asfs.pas @@ -19,6 +19,8 @@ } unit driver.storage.fs.asfs; +{$DEFINE NOT_DANGLING} { Incomplete legacy filesystem; requires removed storagemanagement API. } + interface uses @@ -121,4 +123,6 @@ end; procedure detect_volumes(disk : PStorage_Device) begin -end; \ No newline at end of file +end; + +end. diff --git a/src/driver/storage/vol/driver.storage.vol.table.pas b/src/driver/storage/vol/driver.storage.vol.table.pas index a77f79ea..5f17b938 100644 --- a/src/driver/storage/vol/driver.storage.vol.table.pas +++ b/src/driver/storage/vol/driver.storage.vol.table.pas @@ -19,6 +19,8 @@ unit driver.storage.vol.table; +{$DEFINE NOT_DANGLING} { Unimplemented legacy partition-table API; requires removed storagemanagement. } + interface uses diff --git a/toolchain/architecture.sh b/toolchain/architecture.sh new file mode 100644 index 00000000..ded49047 --- /dev/null +++ b/toolchain/architecture.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Shared source-architecture selection; callers run from the repository root. +# Folder names: x86 = i386, x86_64 = x86_64. +export ASURO_ARCH="${ASURO_ARCH:-x86}" +if [[ ! "$ASURO_ARCH" =~ ^[a-z][a-z0-9_]*$ ]]; then + echo "Invalid ASURO_ARCH: $ASURO_ARCH" >&2 + return 1 +fi +if [[ ! -d "src/arch/$ASURO_ARCH" ]]; then + echo "Architecture sources not found: src/arch/$ASURO_ARCH" >&2 + return 1 +fi diff --git a/toolchain/compile.sh b/toolchain/compile.sh index 485e29a6..621e4201 100644 --- a/toolchain/compile.sh +++ b/toolchain/compile.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +TOOLCHAIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$TOOLCHAIN_DIR/architecture.sh" || exit 1 ERRCOUNT=0 echo " " echo "=======================" @@ -20,14 +22,13 @@ runOrFail() { fi } -TOOLCHAIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - declare -a run_steps=( "compile_stub.sh" "Failed to compile stub!" "compile_vergen.sh" "Versions failed to compile" "compile_lvgl.sh" "Failed to compile LVGL!" "compile_icons.sh" "Failed to generate icons!" "compile_wasuro.sh" "Failed to pull Wasuro!" + "compile_dangling.sh" "Failed to generate dangling unit list!" "compile_sources.sh" "Failed to compile FPC Sources!" "compile_initcalls.sh" "Failed to generate INITFINAL table!" "compile_link.sh" "Failed linking!" @@ -53,4 +54,4 @@ else . "${TOOLCHAIN_DIR}/compile_finish.sh" "success" fi -cd .. \ No newline at end of file +cd .. diff --git a/toolchain/compile_dangling.sh b/toolchain/compile_dangling.sh new file mode 100644 index 00000000..55d05773 --- /dev/null +++ b/toolchain/compile_dangling.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "Generating dangling Pascal units..." +TOOLCHAIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "${TOOLCHAIN_DIR}/.." +source "$TOOLCHAIN_DIR/architecture.sh" +OUT="$(pwd)/src/boot/dangling_units.inc" +# Remove stale output before attempting analysis. Work in a separate temporary +# repository so developer Moderne runs and generated sources cannot contaminate it. +rm -f "$OUT" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +mkdir -p "$WORK/src" "$WORK/compat" +cp -a src/. "$WORK/src/" +cp -a compat/. "$WORK/compat/" +# Do not parse inactive architectures: incomplete ports must not break this build. +for dir in "$WORK/src/arch/"*; do + [[ "$(basename "$dir")" == "$ASURO_ARCH" ]] || rm -rf "$dir" +done +git -C "$WORK" init -q +git -C "$WORK" remote add origin https://gitea.spexeah.com/Spexeah/Asuro.git +PASCAL_JAVA_OPTIONS='-Dpascal.sourceRoots=src,compat -Dpascal.failOnParseError=true' pascal-lst "$WORK" +mod run "$WORK" --recipe org.openrewrite.pascal.boot.FindDanglingUnits \ + --recipe-option "architecture=$ASURO_ARCH" +dangling-include "$WORK" "$OUT" diff --git a/toolchain/compile_sources.sh b/toolchain/compile_sources.sh index b6b86a34..53e78a63 100644 --- a/toolchain/compile_sources.sh +++ b/toolchain/compile_sources.sh @@ -1,13 +1,21 @@ #!/usr/bin/env bash +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/architecture.sh" echo " " echo "=======================" echo " " echo "Compiling FPC Sources..." echo " " -# Build -Fu flags for all directories under src/, wasuro/ and compat/ -FU_PATHS="-Fusrc -Fuwasuro -Fucompat" -for dir in $(find src wasuro compat -type d 2>/dev/null); do - FU_PATHS="$FU_PATHS -Fu$dir" -done +# Keep other architectures out of FPC's unit resolution, including arch/ itself. +FU_PATHS=() +while IFS= read -r -d '' dir; do + FU_PATHS+=("-Fu$dir") +done < <(find src wasuro compat -path src/arch -prune -o -type d -print0 + find "src/arch/$ASURO_ARCH" -type d -print0) -fpc -Aelf -gw -g -gl -n -v0e -O3 -OpPENTIUM3 -Si -Sc -Sg -Xd -CX -XXs -CfSSE -CfSSE2 -Rintel -Pi386 -Tlinux -FElib/ $FU_PATHS src/asuro.pas \ No newline at end of file +if [[ "$ASURO_ARCH" == "x86" ]]; then + fpc -Aelf -gw -g -gl -n -v0e -O3 -OpPENTIUM3 -Si -Sc -Sg -Xd -CX -XXs -CfSSE -CfSSE2 -Rintel -Pi386 -Tlinux -FElib/ "${FU_PATHS[@]}" src/asuro.pas +else + echo "No compiler found for target: $ASURO_ARCH" >&2 + exit 1 +fi -- 2.54.0