Feature/sotrage system overhaul #73

Open
Aaron wants to merge 21 commits from feature/sotrage-system-overhaul into develop
pull from: feature/sotrage-system-overhaul
10 changed files with 636 additions and 4705 deletions
Showing only changes of commit 663bfbb0e4 - Show all commits

No files matched your search

File diff suppressed because it is too large. Load diff
+14
View File
@@ -44,6 +44,7 @@ function inl(port : uint16) : uint32;
procedure io_wait; procedure io_wait;
procedure __SSE_128_memcpy(source : uint32; dest : uint32); procedure __SSE_128_memcpy(source : uint32; dest : uint32);
procedure __REP_MOVSB_memcpy(source : uint32; dest : uint32; size : uint32);
procedure halt_and_catch_fire(); procedure halt_and_catch_fire();
procedure halt_and_dont_catch_fire(); procedure halt_and_dont_catch_fire();
@@ -113,6 +114,19 @@ asm
MOVAPS [EAX], XMM1 MOVAPS [EAX], XMM1
end; end;
procedure __REP_MOVSB_memcpy(source : uint32; dest : uint32; size : uint32); assembler;
asm
push esi
push edi
cld
mov esi, source
mov edi, dest
mov ecx, size
rep movsb
pop edi
pop esi
end;
function getESP : uint32; function getESP : uint32;
var var
tmp: uint32; tmp: uint32;
+28 -9
View File
@@ -42,6 +42,26 @@ function HexCharToDecimal(hex : char) : uint8;
function abs(x : sint32) : uint32; function abs(x : sint32) : uint32;
implementation implementation
{$ifdef CPU386}
uses
arch.x86.util;
const
MEMCPY_X86_SMALL_COPY_THRESHOLD = 16;
{$endif}
procedure scalar_memcpy(source : uint32; dest : uint32; size : uint32);
Review

I believe a scalar memcpy already exists elsewhere? Worth verifying this & using that if it does.

I believe a scalar memcpy already exists elsewhere? Worth verifying this & using that if it does.
var
src, dst : puint8;
i : uint32;
begin
if size = 0 then exit;
for i := 0 to size - 1 do begin
src := puint8(source + i);
dst := puint8(dest + i);
dst^ := src^;
end;
end;
function abs(x : sint32) : uint32; function abs(x : sint32) : uint32;
var var
@@ -123,17 +143,16 @@ begin
end; end;
procedure memcpy(source : uint32; dest : uint32; size : uint32); procedure memcpy(source : uint32; dest : uint32; size : uint32);
var
src, dst : puint8;
i : uint32;
begin begin
if size = 0 then exit; if size = 0 then exit;
for i:=0 to size-1 do begin {$ifdef CPU386}
src:= puint8(source + i); if size < MEMCPY_X86_SMALL_COPY_THRESHOLD then
dst:= puint8(dest + i); scalar_memcpy(source, dest, size)
dst^:= src^; else
end; arch.x86.util.__REP_MOVSB_memcpy(source, dest, size);
{$else}
scalar_memcpy(source, dest, size);
{$endif}
end; end;
function getWord(i : uint32; hi : boolean) : uint16; function getWord(i : uint32; hi : boolean) : uint16;
+57 -17
View File
@@ -53,6 +53,22 @@ type
{ Directory entry core.types } { Directory entry core.types }
TDirectory_Entry_Type = (directoryEntry, fileEntry, mountEntry); TDirectory_Entry_Type = (directoryEntry, fileEntry, mountEntry);
{ Generic directory entry }
TDirectory_Entry = record
fileName : pchar;
entryType : TDirectory_Entry_Type;
fileSize : uint32;
modifiedDate : uint16; { FAT-style: bits 15-9=year-1980, 8-5=month, 4-0=day }
modifiedTime : uint16; { FAT-style: bits 15-11=hour, 10-5=min, 4-0=sec/2 }
attributes : uint8; { FS-specific attribute bits }
end;
TDrive_Error = record
code : uint16;
description : pchar;
recoverable : boolean;
end;
{ Storage error codes — shared across I/O layer, drivers, FS, and VFS } { Storage error codes — shared across I/O layer, drivers, FS, and VFS }
TError = ( TError = (
{ Success } { Success }
@@ -126,6 +142,34 @@ type
PDirectory_Entry = ^TDirectory_Entry; PDirectory_Entry = ^TDirectory_Entry;
PDrive_Error = ^TDrive_Error; PDrive_Error = ^TDrive_Error;
PIORequest = ^TIORequest; PIORequest = ^TIORequest;
PFSFormatParams = ^TFSFormatParams;
TStorageLifecycleEvent = (
sleVolumeAdded,
sleVolumeMounted,
sleVolumeUnmounted,
sleVolumeFormatStarted,
sleVolumeFormatCompleted,
sleVolumeInvalidating,
sleVolumeInvalidated,
sleVolumeRemoved,
sleDeviceAdded,
sleDeviceRemoved
);
TStorageLifecycleCallback = procedure(event : TStorageLifecycleEvent;
device : PStorage_Device;
volume : PStorage_Volume;
error : TError;
userdata : pointer);
TFSFormatParams = record
Version : uint16;
Flags : uint16;
ClusterSize : uint32;
FileCount : uint32;
LabelText : array[0..11] of char;
end;
{ === Driver dispatch type (Phase 2+) === } { === Driver dispatch type (Phase 2+) === }
@@ -142,8 +186,8 @@ type
TIOCallback = procedure(error : TError; userdata : pointer); TIOCallback = procedure(error : TError; userdata : pointer);
{ Filesystem callback core.types } { Filesystem callback core.types }
PPCreateHook = procedure(volume : PStorage_volume; start : uint32; size : uint32; config : puint32); PPCreateHook = procedure(volume : PStorage_volume; start : uint32; size : uint32; config : PFSFormatParams);
PPCreateAsyncHook = procedure(volume : PStorage_volume; start : uint32; size : uint32; config : puint32; callback : TIOCallback; callbackData : pointer); PPCreateAsyncHook = procedure(volume : PStorage_volume; start : uint32; size : uint32; config : PFSFormatParams; callback : TIOCallback; callbackData : pointer);
PPDetectHook = procedure(disk : PStorage_Device); PPDetectHook = procedure(disk : PStorage_Device);
PPCreateDirHook = procedure(volume : PStorage_volume; directory : pchar; dirname : pchar; attributes : uint32; status : puint32); PPCreateDirHook = procedure(volume : PStorage_volume; directory : pchar; dirname : pchar; attributes : uint32; status : puint32);
PPReadDirHook = function(volume : PStorage_volume; directory : pchar; status : puint32) : PLinkedListBase; PPReadDirHook = function(volume : PStorage_volume; directory : pchar; status : puint32) : PLinkedListBase;
@@ -259,6 +303,7 @@ type
{ Per-file open/close — nil if FS does not cache per-file metadata } { Per-file open/close — nil if FS does not cache per-file metadata }
openFileCallback : PPOpenFileHook; openFileCallback : PPOpenFileHook;
closeFileCallback : PPCloseFileHook; closeFileCallback : PPCloseFileHook;
formatParamFlags : uint32;
end; end;
{ Generic storage volume } { Generic storage volume }
@@ -271,24 +316,19 @@ type
freeSectors : uint32; freeSectors : uint32;
filesystem : PFilesystem; filesystem : PFilesystem;
isBootDrive : boolean; isBootDrive : boolean;
lifecycleFlags : uint32;
fsPrivate : pointer; { FS-driver private data (e.g. PFATCache for FAT32) } fsPrivate : pointer; { FS-driver private data (e.g. PFATCache for FAT32) }
end; end;
{ Generic directory entry } const
TDirectory_Entry = record FS_FORMAT_PARAM_CLUSTER_SIZE = 1;
fileName : pchar; FS_FORMAT_PARAM_FILE_COUNT = 2;
entryType : TDirectory_Entry_Type; FS_FORMAT_PARAM_LABEL = 4;
fileSize : uint32; FS_FORMAT_PARAMS_VERSION_1 = 1;
modifiedDate : uint16; { FAT-style: bits 15-9=year-1980, 8-5=month, 4-0=day } STORAGE_VOLUME_FLAG_MOUNTED = 1;
modifiedTime : uint16; { FAT-style: bits 15-11=hour, 10-5=min, 4-0=sec/2 } STORAGE_VOLUME_FLAG_FORMATTING = 2;
attributes : uint8; { FS-specific attribute bits } STORAGE_VOLUME_FLAG_INVALIDATING = 4;
end; STORAGE_VOLUME_FLAG_REMOVED = 8;
TDrive_Error = record
code : uint16;
description : pchar;
recoverable : boolean;
end;
implementation implementation
+38 -3
View File
1
@@ -558,6 +558,7 @@ begin
debug.tracer.push_trace('driver.storage.vfs.InvalidateVolume.busy'); debug.tracer.push_trace('driver.storage.vfs.InvalidateVolume.busy');
exit; exit;
end; end;
driver.storage.vol.mgr.volume_set_mounted(vol, false);
{ 1. Evict all directory cache entries for this volume } { 1. Evict all directory cache entries for this volume }
DirCache_Invalidate(vol, nil); DirCache_Invalidate(vol, nil);
@@ -1163,6 +1164,7 @@ begin
core.ds.hashmap.add(ht, stringCopy(mountName), void(mountObj)); core.ds.hashmap.add(ht, stringCopy(mountName), void(mountObj));
mountVolume := pvRegistered; mountVolume := pvRegistered;
driver.storage.vol.mgr.volume_set_mounted(volume, true);
kfree(void(parentPath)); kfree(void(parentPath));
STRLL_Free(splitPath); STRLL_Free(splitPath);
@@ -2434,6 +2436,7 @@ end;
procedure DeleteFileAsync(Path : pchar; Error : PError; Callback : TIOCallback; CallbackData : pointer); procedure DeleteFileAsync(Path : pchar; Error : PError; Callback : TIOCallback; CallbackData : pointer);
var var
ctx : PVFSPathAsyncCtx; ctx : PVFSPathAsyncCtx;
worker : proc.types.PProcessContext;
begin begin
debug.tracer.push_trace('driver.storage.vfs.DeleteFileAsync.enter'); debug.tracer.push_trace('driver.storage.vfs.DeleteFileAsync.enter');
if Error <> nil then Error^ := eUnknown; if Error <> nil then Error^ := eUnknown;
@@ -2462,7 +2465,15 @@ begin
ctx^.IsDirectory := false; ctx^.IsDirectory := false;
ctx^.UserCallback := Callback; ctx^.UserCallback := Callback;
ctx^.UserData := CallbackData; ctx^.UserData := CallbackData;
proc.mgr.create('vfs.delf', @vfs_pathop_worker, void(ctx), 1); worker := proc.mgr.create('vfs.delf', @vfs_pathop_worker, void(ctx), 1);
if worker = nil then begin
if ctx^.Path <> nil then
kfree(void(ctx^.Path));
kfree(void(ctx));
if Error <> nil then Error^ := eOutOfMemory;
if Callback <> nil then Callback(eOutOfMemory, CallbackData);
exit;
end;
debug.tracer.push_trace('driver.storage.vfs.DeleteFileAsync.exit'); debug.tracer.push_trace('driver.storage.vfs.DeleteFileAsync.exit');
end; end;
@@ -2470,6 +2481,7 @@ end;
procedure DeleteDirectoryAsync(Path : pchar; Error : PError; Callback : TIOCallback; CallbackData : pointer); procedure DeleteDirectoryAsync(Path : pchar; Error : PError; Callback : TIOCallback; CallbackData : pointer);
var var
ctx : PVFSPathAsyncCtx; ctx : PVFSPathAsyncCtx;
worker : proc.types.PProcessContext;
begin begin
debug.tracer.push_trace('driver.storage.vfs.DeleteDirectoryAsync.enter'); debug.tracer.push_trace('driver.storage.vfs.DeleteDirectoryAsync.enter');
if Error <> nil then Error^ := eUnknown; if Error <> nil then Error^ := eUnknown;
@@ -2498,7 +2510,15 @@ begin
ctx^.IsDirectory := true; ctx^.IsDirectory := true;
ctx^.UserCallback := Callback; ctx^.UserCallback := Callback;
ctx^.UserData := CallbackData; ctx^.UserData := CallbackData;
proc.mgr.create('vfs.deld', @vfs_pathop_worker, void(ctx), 1); worker := proc.mgr.create('vfs.deld', @vfs_pathop_worker, void(ctx), 1);
if worker = nil then begin
if ctx^.Path <> nil then
kfree(void(ctx^.Path));
kfree(void(ctx));
if Error <> nil then Error^ := eOutOfMemory;
if Callback <> nil then Callback(eOutOfMemory, CallbackData);
exit;
end;
debug.tracer.push_trace('driver.storage.vfs.DeleteDirectoryAsync.exit'); debug.tracer.push_trace('driver.storage.vfs.DeleteDirectoryAsync.exit');
end; end;
@@ -2561,6 +2581,7 @@ procedure GetDirectoryListingAsync(Path : pchar; ResultMap : PPHashMap; Callback
var var
ctx : PVFSDirAsyncCtx; ctx : PVFSDirAsyncCtx;
absPath : pchar; absPath : pchar;
worker : proc.types.PProcessContext;
begin begin
debug.tracer.push_trace('driver.storage.vfs.GetDirectoryListingAsync.enter'); debug.tracer.push_trace('driver.storage.vfs.GetDirectoryListingAsync.enter');
if ResultMap <> nil then ResultMap^ := nil; if ResultMap <> nil then ResultMap^ := nil;
@@ -2573,12 +2594,26 @@ begin
absPath := MakeAbsolutePath(Path); absPath := MakeAbsolutePath(Path);
ctx := PVFSDirAsyncCtx(kalloc(sizeof(TVFSDirAsyncCtx))); ctx := PVFSDirAsyncCtx(kalloc(sizeof(TVFSDirAsyncCtx)));
if ctx = nil then begin
if absPath <> nil then
kfree(void(absPath));
if Callback <> nil then Callback(eOutOfMemory, CallbackData);
exit;
end;
memset(uint32(ctx), 0, sizeof(TVFSDirAsyncCtx));
ctx^.Path := absPath; ctx^.Path := absPath;
ctx^.ResultMap := ResultMap; ctx^.ResultMap := ResultMap;
ctx^.UserCallback := Callback; ctx^.UserCallback := Callback;
ctx^.UserData := CallbackData; ctx^.UserData := CallbackData;
proc.mgr.create('vfs.dirls', @vfs_dirlist_worker, void(ctx), 1); worker := proc.mgr.create('vfs.dirls', @vfs_dirlist_worker, void(ctx), 1);
if worker = nil then begin
if ctx^.Path <> nil then
kfree(void(ctx^.Path));
kfree(void(ctx));
if Callback <> nil then Callback(eOutOfMemory, CallbackData);
exit;
end;
debug.tracer.push_trace('driver.storage.vfs.GetDirectoryListingAsync.exit'); debug.tracer.push_trace('driver.storage.vfs.GetDirectoryListingAsync.exit');
end; end;
File diff suppressed because it is too large. Load diff
@@ -50,6 +50,7 @@ begin
filesystem.closeFileCallback := @FAT32CloseFile; filesystem.closeFileCallback := @FAT32CloseFile;
filesystem.createDirAsyncCallback := nil; filesystem.createDirAsyncCallback := nil;
filesystem.readDirAsyncCallback := nil; filesystem.readDirAsyncCallback := nil;
filesystem.formatParamFlags := FS_FORMAT_PARAM_CLUSTER_SIZE;
driver.storage.fs.mgr.register_filesystem(@filesystem); driver.storage.fs.mgr.register_filesystem(@filesystem);
end; end;
@@ -63,7 +63,7 @@ var
procedure init; procedure init;
procedure create_volume(volume : PStorage_Volume; sectors : uint32; start : uint32; config : puint32); procedure create_volume(volume : PStorage_Volume; sectors : uint32; start : uint32; config : PFSFormatParams);
procedure detect_volumes(disk : PStorage_Device); procedure detect_volumes(disk : PStorage_Device);
function read_directory(volume : PStorage_Volume; directory : pchar; status : PuInt32) : PLinkedListBase; function read_directory(volume : PStorage_Volume; directory : pchar; status : PuInt32) : PLinkedListBase;
procedure write_directory(volume : PStorage_Volume; directory : pchar; status : PuInt32); procedure write_directory(volume : PStorage_Volume; directory : pchar; status : PuInt32);
@@ -72,7 +72,7 @@ procedure read_file(volume : PStorage_Volume; fileName : pchar; data : PuInt32;
implementation implementation
procedure create_volume(volume : PStorage_Volume; sectors : uint32; start : uint32; config : puint32); procedure create_volume(volume : PStorage_Volume; sectors : uint32; start : uint32; config : PFSFormatParams);
var var
info : PDisk_Info; info : PDisk_Info;
entryTable : PFile_Entry; entryTable : PFile_Entry;
@@ -102,8 +102,11 @@ begin
info^.fileCount := 1000; info^.fileCount := 1000;
info^.signature := $0B00B1E5; info^.signature := $0B00B1E5;
if config^ <> 0 then begin if (config <> nil) and
info^.fileCount := config^; (config^.Version = FS_FORMAT_PARAMS_VERSION_1) and
((config^.Flags and FS_FORMAT_PARAM_FILE_COUNT) <> 0) and
(config^.FileCount <> 0) then begin
info^.fileCount := config^.FileCount;
end; end;
driver.storage.mgr.storage_write(volume^.device, start, 1, PuInt32(info));// what happens if buffer is smaller than 512? driver.storage.mgr.storage_write(volume^.device, start, 1, PuInt32(info));// what happens if buffer is smaller than 512?
@@ -975,6 +978,7 @@ begin
filesystem.createcallback:= @create_volume; filesystem.createcallback:= @create_volume;
filesystem.detectcallback:= @detect_volumes; filesystem.detectcallback:= @detect_volumes;
filesystem.identifyCallback:= @identify_volume; filesystem.identifyCallback:= @identify_volume;
filesystem.formatParamFlags := FS_FORMAT_PARAM_FILE_COUNT;
// filesystem.writecallback:= @writeFile; // filesystem.writecallback:= @writeFile;
// filesystem.readcallback := @readFile; // filesystem.readcallback := @readFile;
File diff suppressed because it is too large. Load diff
+51
View File
@@ -119,11 +119,17 @@ var
prio : uint8; prio : uint8;
begin begin
push_trace('proc.mgr.create'); push_trace('proc.mgr.create');
create := nil;
if priority = 0 then prio := 1 else prio := priority; if priority = 0 then prio := 1 else prio := priority;
{ Allocate process context } { Allocate process context }
ctx := PProcessContext(kalloc(SizeOf(TProcessContext))); ctx := PProcessContext(kalloc(SizeOf(TProcessContext)));
if ctx = nil then begin
io.syslog.logln('PROCMGR', 'Failed to allocate process context.');
pop_trace;
exit;
end;
memset(uint32(ctx), 0, SizeOf(TProcessContext)); memset(uint32(ctx), 0, SizeOf(TProcessContext));
{ Identity } { Identity }
@@ -154,15 +160,45 @@ begin
{ Resources } { Resources }
ctx^.Resources := void(DL_New(SizeOf(TResourceBinding))); ctx^.Resources := void(DL_New(SizeOf(TResourceBinding)));
if ctx^.Resources = nil then begin
io.syslog.logln('PROCMGR', 'Failed to allocate resource list.');
kfree(void(ctx));
pop_trace;
exit;
end;
{ Per-process file descriptor table } { Per-process file descriptor table }
ctx^.FDTable := void(fd_table_new); ctx^.FDTable := void(fd_table_new);
if ctx^.FDTable = nil then begin
io.syslog.logln('PROCMGR', 'Failed to allocate FD table.');
DL_Free(PDList(ctx^.Resources));
kfree(void(ctx));
pop_trace;
exit;
Review

This should probably be genericised into an Assert or similar. Lots of duplicate code for exception checking.

This should probably be genericised into an Assert or similar. Lots of duplicate code for exception checking.
end;
{ Per-process working directory } { Per-process working directory }
ctx^.Cwd := stringCopy('/'); ctx^.Cwd := stringCopy('/');
if ctx^.Cwd = nil then begin
io.syslog.logln('PROCMGR', 'Failed to allocate working directory.');
fd_table_free(PFDTable(ctx^.FDTable));
DL_Free(PDList(ctx^.Resources));
kfree(void(ctx));
pop_trace;
exit;
end;
{ Allocate per-process core.version stack } { Allocate per-process core.version stack }
stack := kalloc(PROCESS_STACK_SIZE); stack := kalloc(PROCESS_STACK_SIZE);
if stack = nil then begin
io.syslog.logln('PROCMGR', 'Failed to allocate process stack.');
kfree(void(ctx^.Cwd));
fd_table_free(PFDTable(ctx^.FDTable));
DL_Free(PDList(ctx^.Resources));
kfree(void(ctx));
pop_trace;
exit;
end;
ctx^.StackBase := stack; ctx^.StackBase := stack;
ctx^.StackTop := uint32(stack) + PROCESS_STACK_SIZE; ctx^.StackTop := uint32(stack) + PROCESS_STACK_SIZE;
@@ -215,6 +251,21 @@ begin
{ Add to process table } { Add to process table }
asm pushf; cli end; asm pushf; cli end;
slotPtr := DL_Add(Processes); slotPtr := DL_Add(Processes);
if slotPtr = nil then begin
asm popf end;
io.syslog.logln('PROCMGR', 'Failed to allocate process table slot.');
if ctx^.StackBase <> nil then
kfree(ctx^.StackBase);
if ctx^.Cwd <> nil then
kfree(void(ctx^.Cwd));
if ctx^.FDTable <> nil then
fd_table_free(PFDTable(ctx^.FDTable));
if ctx^.Resources <> nil then
DL_Free(PDList(ctx^.Resources));
kfree(void(ctx));
pop_trace;
exit;
end;
slotPtr^ := uint32(ctx); slotPtr^ := uint32(ctx);
{ Mark as ready for scheduling } { Mark as ready for scheduling }