Feature/sotrage system overhaul #73

Open
Aaron wants to merge 21 commits from feature/sotrage-system-overhaul into develop
pull from: feature/sotrage-system-overhaul
10 changed files with 636 additions and 4705 deletions
Showing only changes of commit 663bfbb0e4 - Show all commits

No files matched your search

File diff suppressed because it is too large. Load diff
+14
View File
@@ -44,6 +44,7 @@ function inl(port : uint16) : uint32;
procedure io_wait;
procedure __SSE_128_memcpy(source : uint32; dest : uint32);
procedure __REP_MOVSB_memcpy(source : uint32; dest : uint32; size : uint32);
procedure halt_and_catch_fire();
procedure halt_and_dont_catch_fire();
@@ -113,6 +114,19 @@ asm
MOVAPS [EAX], XMM1
end;
procedure __REP_MOVSB_memcpy(source : uint32; dest : uint32; size : uint32); assembler;
asm
push esi
push edi
cld
mov esi, source
mov edi, dest
mov ecx, size
rep movsb
pop edi
pop esi
end;
function getESP : uint32;
var
tmp: uint32;
+28 -9
View File
@@ -42,6 +42,26 @@ function HexCharToDecimal(hex : char) : uint8;
function abs(x : sint32) : uint32;
implementation
{$ifdef CPU386}
uses
arch.x86.util;
const
MEMCPY_X86_SMALL_COPY_THRESHOLD = 16;
{$endif}
procedure scalar_memcpy(source : uint32; dest : uint32; size : uint32);
Review

I believe a scalar memcpy already exists elsewhere? Worth verifying this & using that if it does.

I believe a scalar memcpy already exists elsewhere? Worth verifying this & using that if it does.
var
src, dst : puint8;
i : uint32;
begin
if size = 0 then exit;
for i := 0 to size - 1 do begin
src := puint8(source + i);
dst := puint8(dest + i);
dst^ := src^;
end;
end;
function abs(x : sint32) : uint32;
var
@@ -123,17 +143,16 @@ begin
end;
procedure memcpy(source : uint32; dest : uint32; size : uint32);
var
src, dst : puint8;
i : uint32;
begin
if size = 0 then exit;
for i:=0 to size-1 do begin
src:= puint8(source + i);
dst:= puint8(dest + i);
dst^:= src^;
end;
{$ifdef CPU386}
if size < MEMCPY_X86_SMALL_COPY_THRESHOLD then
scalar_memcpy(source, dest, size)
else
arch.x86.util.__REP_MOVSB_memcpy(source, dest, size);
{$else}
scalar_memcpy(source, dest, size);
{$endif}
end;
function getWord(i : uint32; hi : boolean) : uint16;
+57 -17
View File
@@ -53,6 +53,22 @@ type
{ Directory entry core.types }
TDirectory_Entry_Type = (directoryEntry, fileEntry, mountEntry);
{ Generic directory entry }
TDirectory_Entry = record
fileName : pchar;
entryType : TDirectory_Entry_Type;
fileSize : uint32;
modifiedDate : uint16; { FAT-style: bits 15-9=year-1980, 8-5=month, 4-0=day }
modifiedTime : uint16; { FAT-style: bits 15-11=hour, 10-5=min, 4-0=sec/2 }
attributes : uint8; { FS-specific attribute bits }
end;
TDrive_Error = record
code : uint16;
description : pchar;
recoverable : boolean;
end;
{ Storage error codes — shared across I/O layer, drivers, FS, and VFS }
TError = (
{ Success }
@@ -126,6 +142,34 @@ type
PDirectory_Entry = ^TDirectory_Entry;
PDrive_Error = ^TDrive_Error;
PIORequest = ^TIORequest;
PFSFormatParams = ^TFSFormatParams;
TStorageLifecycleEvent = (
sleVolumeAdded,
sleVolumeMounted,
sleVolumeUnmounted,
sleVolumeFormatStarted,
sleVolumeFormatCompleted,
sleVolumeInvalidating,
sleVolumeInvalidated,
sleVolumeRemoved,
sleDeviceAdded,
sleDeviceRemoved
);
TStorageLifecycleCallback = procedure(event : TStorageLifecycleEvent;
device : PStorage_Device;
volume : PStorage_Volume;
error : TError;
userdata : pointer);
TFSFormatParams = record
Version : uint16;
Flags : uint16;
ClusterSize : uint32;
FileCount : uint32;
LabelText : array[0..11] of char;
end;
{ === Driver dispatch type (Phase 2+) === }
@@ -142,8 +186,8 @@ type
TIOCallback = procedure(error : TError; userdata : pointer);
{ Filesystem callback core.types }
PPCreateHook = procedure(volume : PStorage_volume; start : uint32; size : uint32; config : puint32);
PPCreateAsyncHook = procedure(volume : PStorage_volume; start : uint32; size : uint32; config : puint32; callback : TIOCallback; callbackData : pointer);
PPCreateHook = procedure(volume : PStorage_volume; start : uint32; size : uint32; config : PFSFormatParams);
PPCreateAsyncHook = procedure(volume : PStorage_volume; start : uint32; size : uint32; config : PFSFormatParams; callback : TIOCallback; callbackData : pointer);
PPDetectHook = procedure(disk : PStorage_Device);
PPCreateDirHook = procedure(volume : PStorage_volume; directory : pchar; dirname : pchar; attributes : uint32; status : puint32);
PPReadDirHook = function(volume : PStorage_volume; directory : pchar; status : puint32) : PLinkedListBase;
@@ -259,6 +303,7 @@ type
{ Per-file open/close — nil if FS does not cache per-file metadata }
openFileCallback : PPOpenFileHook;
closeFileCallback : PPCloseFileHook;
formatParamFlags : uint32;
end;
{ Generic storage volume }
@@ -271,24 +316,19 @@ type
freeSectors : uint32;
filesystem : PFilesystem;
isBootDrive : boolean;
lifecycleFlags : uint32;
fsPrivate : pointer; { FS-driver private data (e.g. PFATCache for FAT32) }
end;
{ Generic directory entry }
TDirectory_Entry = record
fileName : pchar;
entryType : TDirectory_Entry_Type;
fileSize : uint32;
modifiedDate : uint16; { FAT-style: bits 15-9=year-1980, 8-5=month, 4-0=day }
modifiedTime : uint16; { FAT-style: bits 15-11=hour, 10-5=min, 4-0=sec/2 }
attributes : uint8; { FS-specific attribute bits }
end;
TDrive_Error = record
code : uint16;
description : pchar;
recoverable : boolean;
end;
const
FS_FORMAT_PARAM_CLUSTER_SIZE = 1;
FS_FORMAT_PARAM_FILE_COUNT = 2;
FS_FORMAT_PARAM_LABEL = 4;
FS_FORMAT_PARAMS_VERSION_1 = 1;
STORAGE_VOLUME_FLAG_MOUNTED = 1;
STORAGE_VOLUME_FLAG_FORMATTING = 2;
STORAGE_VOLUME_FLAG_INVALIDATING = 4;
STORAGE_VOLUME_FLAG_REMOVED = 8;
implementation
+38 -3
View File
1
@@ -558,6 +558,7 @@ begin
debug.tracer.push_trace('driver.storage.vfs.InvalidateVolume.busy');
exit;
end;
driver.storage.vol.mgr.volume_set_mounted(vol, false);
{ 1. Evict all directory cache entries for this volume }
DirCache_Invalidate(vol, nil);
@@ -1163,6 +1164,7 @@ begin
core.ds.hashmap.add(ht, stringCopy(mountName), void(mountObj));
mountVolume := pvRegistered;
driver.storage.vol.mgr.volume_set_mounted(volume, true);
kfree(void(parentPath));
STRLL_Free(splitPath);
@@ -2434,6 +2436,7 @@ end;
procedure DeleteFileAsync(Path : pchar; Error : PError; Callback : TIOCallback; CallbackData : pointer);
var
ctx : PVFSPathAsyncCtx;
worker : proc.types.PProcessContext;
begin
debug.tracer.push_trace('driver.storage.vfs.DeleteFileAsync.enter');
if Error <> nil then Error^ := eUnknown;
@@ -2462,7 +2465,15 @@ begin
ctx^.IsDirectory := false;
ctx^.UserCallback := Callback;
ctx^.UserData := CallbackData;
proc.mgr.create('vfs.delf', @vfs_pathop_worker, void(ctx), 1);
worker := proc.mgr.create('vfs.delf', @vfs_pathop_worker, void(ctx), 1);
if worker = nil then begin
if ctx^.Path <> nil then
kfree(void(ctx^.Path));
kfree(void(ctx));
if Error <> nil then Error^ := eOutOfMemory;
if Callback <> nil then Callback(eOutOfMemory, CallbackData);
exit;
end;
debug.tracer.push_trace('driver.storage.vfs.DeleteFileAsync.exit');
end;
@@ -2470,6 +2481,7 @@ end;
procedure DeleteDirectoryAsync(Path : pchar; Error : PError; Callback : TIOCallback; CallbackData : pointer);
var
ctx : PVFSPathAsyncCtx;
worker : proc.types.PProcessContext;
begin
debug.tracer.push_trace('driver.storage.vfs.DeleteDirectoryAsync.enter');
if Error <> nil then Error^ := eUnknown;
@@ -2498,7 +2510,15 @@ begin
ctx^.IsDirectory := true;
ctx^.UserCallback := Callback;
ctx^.UserData := CallbackData;
proc.mgr.create('vfs.deld', @vfs_pathop_worker, void(ctx), 1);
worker := proc.mgr.create('vfs.deld', @vfs_pathop_worker, void(ctx), 1);
if worker = nil then begin
if ctx^.Path <> nil then
kfree(void(ctx^.Path));
kfree(void(ctx));
if Error <> nil then Error^ := eOutOfMemory;
if Callback <> nil then Callback(eOutOfMemory, CallbackData);
exit;
end;
debug.tracer.push_trace('driver.storage.vfs.DeleteDirectoryAsync.exit');
end;
@@ -2561,6 +2581,7 @@ procedure GetDirectoryListingAsync(Path : pchar; ResultMap : PPHashMap; Callback
var
ctx : PVFSDirAsyncCtx;
absPath : pchar;
worker : proc.types.PProcessContext;
begin
debug.tracer.push_trace('driver.storage.vfs.GetDirectoryListingAsync.enter');
if ResultMap <> nil then ResultMap^ := nil;
@@ -2573,12 +2594,26 @@ begin
absPath := MakeAbsolutePath(Path);
ctx := PVFSDirAsyncCtx(kalloc(sizeof(TVFSDirAsyncCtx)));
if ctx = nil then begin
if absPath <> nil then
kfree(void(absPath));
if Callback <> nil then Callback(eOutOfMemory, CallbackData);
exit;
end;
memset(uint32(ctx), 0, sizeof(TVFSDirAsyncCtx));
ctx^.Path := absPath;
ctx^.ResultMap := ResultMap;
ctx^.UserCallback := Callback;
ctx^.UserData := CallbackData;
proc.mgr.create('vfs.dirls', @vfs_dirlist_worker, void(ctx), 1);
worker := proc.mgr.create('vfs.dirls', @vfs_dirlist_worker, void(ctx), 1);
if worker = nil then begin
if ctx^.Path <> nil then
kfree(void(ctx^.Path));
kfree(void(ctx));
if Callback <> nil then Callback(eOutOfMemory, CallbackData);
exit;
end;
debug.tracer.push_trace('driver.storage.vfs.GetDirectoryListingAsync.exit');
end;
File diff suppressed because it is too large. Load diff
@@ -50,6 +50,7 @@ begin
filesystem.closeFileCallback := @FAT32CloseFile;
filesystem.createDirAsyncCallback := nil;
filesystem.readDirAsyncCallback := nil;
filesystem.formatParamFlags := FS_FORMAT_PARAM_CLUSTER_SIZE;
driver.storage.fs.mgr.register_filesystem(@filesystem);
end;
@@ -63,7 +63,7 @@ var
procedure init;
procedure create_volume(volume : PStorage_Volume; sectors : uint32; start : uint32; config : puint32);
procedure create_volume(volume : PStorage_Volume; sectors : uint32; start : uint32; config : PFSFormatParams);
procedure detect_volumes(disk : PStorage_Device);
function read_directory(volume : PStorage_Volume; directory : pchar; status : PuInt32) : PLinkedListBase;
procedure write_directory(volume : PStorage_Volume; directory : pchar; status : PuInt32);
@@ -72,7 +72,7 @@ procedure read_file(volume : PStorage_Volume; fileName : pchar; data : PuInt32;
implementation
procedure create_volume(volume : PStorage_Volume; sectors : uint32; start : uint32; config : puint32);
procedure create_volume(volume : PStorage_Volume; sectors : uint32; start : uint32; config : PFSFormatParams);
var
info : PDisk_Info;
entryTable : PFile_Entry;
@@ -102,8 +102,11 @@ begin
info^.fileCount := 1000;
info^.signature := $0B00B1E5;
if config^ <> 0 then begin
info^.fileCount := config^;
if (config <> nil) and
(config^.Version = FS_FORMAT_PARAMS_VERSION_1) and
((config^.Flags and FS_FORMAT_PARAM_FILE_COUNT) <> 0) and
(config^.FileCount <> 0) then begin
info^.fileCount := config^.FileCount;
end;
driver.storage.mgr.storage_write(volume^.device, start, 1, PuInt32(info));// what happens if buffer is smaller than 512?
@@ -975,6 +978,7 @@ begin
filesystem.createcallback:= @create_volume;
filesystem.detectcallback:= @detect_volumes;
filesystem.identifyCallback:= @identify_volume;
filesystem.formatParamFlags := FS_FORMAT_PARAM_FILE_COUNT;
// filesystem.writecallback:= @writeFile;
// filesystem.readcallback := @readFile;
File diff suppressed because it is too large. Load diff
+51
View File
@@ -119,11 +119,17 @@ var
prio : uint8;
begin
push_trace('proc.mgr.create');
create := nil;
if priority = 0 then prio := 1 else prio := priority;
{ Allocate process context }
ctx := PProcessContext(kalloc(SizeOf(TProcessContext)));
if ctx = nil then begin
io.syslog.logln('PROCMGR', 'Failed to allocate process context.');
pop_trace;
exit;
end;
memset(uint32(ctx), 0, SizeOf(TProcessContext));
{ Identity }
@@ -154,15 +160,45 @@ begin
{ Resources }
ctx^.Resources := void(DL_New(SizeOf(TResourceBinding)));
if ctx^.Resources = nil then begin
io.syslog.logln('PROCMGR', 'Failed to allocate resource list.');
kfree(void(ctx));
pop_trace;
exit;
end;
{ Per-process file descriptor table }
ctx^.FDTable := void(fd_table_new);
if ctx^.FDTable = nil then begin
io.syslog.logln('PROCMGR', 'Failed to allocate FD table.');
DL_Free(PDList(ctx^.Resources));
kfree(void(ctx));
pop_trace;
exit;
Review

This should probably be genericised into an Assert or similar. Lots of duplicate code for exception checking.

This should probably be genericised into an Assert or similar. Lots of duplicate code for exception checking.
end;
{ Per-process working directory }
ctx^.Cwd := stringCopy('/');
if ctx^.Cwd = nil then begin
io.syslog.logln('PROCMGR', 'Failed to allocate working directory.');
fd_table_free(PFDTable(ctx^.FDTable));
DL_Free(PDList(ctx^.Resources));
kfree(void(ctx));
pop_trace;
exit;
end;
{ Allocate per-process core.version stack }
stack := kalloc(PROCESS_STACK_SIZE);
if stack = nil then begin
io.syslog.logln('PROCMGR', 'Failed to allocate process stack.');
kfree(void(ctx^.Cwd));
fd_table_free(PFDTable(ctx^.FDTable));
DL_Free(PDList(ctx^.Resources));
kfree(void(ctx));
pop_trace;
exit;
end;
ctx^.StackBase := stack;
ctx^.StackTop := uint32(stack) + PROCESS_STACK_SIZE;
@@ -215,6 +251,21 @@ begin
{ Add to process table }
asm pushf; cli end;
slotPtr := DL_Add(Processes);
if slotPtr = nil then begin
asm popf end;
io.syslog.logln('PROCMGR', 'Failed to allocate process table slot.');
if ctx^.StackBase <> nil then
kfree(ctx^.StackBase);
if ctx^.Cwd <> nil then
kfree(void(ctx^.Cwd));
if ctx^.FDTable <> nil then
fd_table_free(PFDTable(ctx^.FDTable));
if ctx^.Resources <> nil then
DL_Free(PDList(ctx^.Resources));
kfree(void(ctx));
pop_trace;
exit;
end;
slotPtr^ := uint32(ctx);
{ Mark as ready for scheduling }