Full Rewrite

.gitignore

@@ -1,9 +1,10 @@
 ## Ignore Visual Studio temporary files, build results, and
 ## files generated by popular Visual Studio add-ons.
 ##
-## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
+## Get latest from `dotnet new gitignore`
 
-Simple/
+# dotenv files
+.env
 
 # User-specific files
 *.rsuser
@@ -31,7 +32,6 @@ x86/
 bld/
 [Bb]in/
 [Oo]bj/
-[Oo]ut/
 [Ll]og/
 [Ll]ogs/
 
@@ -60,11 +60,14 @@ dlldata.c
 # Benchmark Results
 BenchmarkDotNet.Artifacts/
 
-# .NET Core
+# .NET
 project.lock.json
 project.fragment.lock.json
 artifacts/
 
+# Tye
+.tye/
+
 # ASP.NET Scaffolding
 ScaffoldingReadMe.txt
 
@@ -85,6 +88,8 @@ StyleCopReport.xml
 *.pgc
 *.pgd
 *.rsp
+# but not Directory.Build.rsp, as it configures directory-level build defaults
+!Directory.Build.rsp
 *.sbr
 *.tlb
 *.tli
@@ -93,6 +98,7 @@ StyleCopReport.xml
 *.tmp_proj
 *_wpftmp.csproj
 *.log
+*.tlog
 *.vspscc
 *.vssscc
 .builds
@@ -296,6 +302,17 @@ node_modules/
 # Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
 *.vbw
 
+# Visual Studio 6 auto-generated project file (contains which files were open etc.)
+*.vbp
+
+# Visual Studio 6 workspace and project file (working project files containing files to include in project)
+*.dsw
+*.dsp
+
+# Visual Studio 6 technical files
+*.ncb
+*.aps
+
 # Visual Studio LightSwitch build output
 **/*.HTMLClient/GeneratedArtifacts
 **/*.DesktopClient/GeneratedArtifacts
@@ -352,6 +369,9 @@ ASALocalRun/
 # Local History for Visual Studio
 .localhistory/
 
+# Visual Studio History (VSHistory) files
+.vshistory/
+
 # BeatPulse healthcheck temp database
 healthchecksdb
 
@@ -363,4 +383,100 @@ MigrationBackup/
 
 # Fody - auto-generated XML schema
 FodyWeavers.xsd
-/Simple/Simple.csproj
+
+# VS Code files for those working on multiple tools
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+
+# Local History for Visual Studio Code
+.history/
+
+# Windows Installer files from build outputs
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# JetBrains Rider
+*.sln.iml
+.idea/
+
+##
+## Visual studio for Mac
+##
+
+
+# globs
+Makefile.in
+*.userprefs
+*.usertasks
+config.make
+config.status
+aclocal.m4
+install-sh
+autom4te.cache/
+*.tar.gz
+tarballs/
+test-results/
+
+# content below from: https://github.com/github/gitignore/blob/main/Global/macOS.gitignore
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+# content below from: https://github.com/github/gitignore/blob/main/Global/Windows.gitignore
+# Windows thumbnail cache files
+Thumbs.db
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+# Vim temporary swap files
+*.swp

Directory.Build.props

@@ -2,6 +2,7 @@
   <PropertyGroup>
     <DotNetVersion>net10.0</DotNetVersion>
     <Nullable>enable</Nullable>
+    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
   </PropertyGroup>
 
   <ItemGroup>

MachineLearning.Benchmarks/MachineLearning.Benchmarks.csproj → ML.Benchy/ML.Benchy.csproj

@@ -8,12 +8,11 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="BenchmarkDotNet" Version="0.15.8" />
+    <ProjectReference Include="..\ML.Runner\ML.Runner.csproj" />
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\MachineLearning.Training\MachineLearning.Training.csproj" />
-    <ProjectReference Include="..\MachineLearning.Samples\MachineLearning.Samples.csproj" />
+    <PackageReference Include="BenchmarkDotNet" Version="0.15.8" />
   </ItemGroup>
 
 </Project>

ML.Benchy/Program.cs

@@ -0,0 +1,44 @@
+using System.Buffers;
+using Ametrin.Numerics;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Running;
+using ML.Core.Training;
+using Weight = float;
+
+BenchmarkRunner.Run<Benchmarks>();
+
+[MemoryDiagnoser(false)]
+public class Benchmarks
+{
+    [Params(512)]
+    public int Size { get; set; }
+    private Vector logits;
+    private Vector expected;
+    private Vector destination;
+
+    private AdamOptimizer optimizer = new() { LearningRate = 0.01f };
+
+
+    [GlobalSetup]
+    public void Setup()
+    {
+        logits = Vector.Create(Size);
+        logits.Uniform(-1, 1, new Random(43));
+        expected = Vector.Create(Size);
+        expected.Uniform(-1, 1, new Random(68));
+        destination = Vector.Create(Size);
+        optimizer.Init();
+    }
+
+    [Benchmark]
+    public void Delegates()
+    {
+        // SpanOperations.MapTo(logits.AsSpan(), expected.AsSpan(), destination.AsSpan(), optimizer.WeightReduction, optimizer.WeightReduction);
+    }
+
+    [Benchmark]
+    public void Static()
+    {
+        // SpanOperations.MapTo(optimizer.WeightReductionOperation, logits.AsSpan(), expected.AsSpan(), destination.AsSpan());
+    }
+}

ML.Core/Attributes.cs

@@ -0,0 +1,18 @@
+namespace ML.Core.Attributes;
+
+#pragma warning disable CS9113 // Parameter is unread. only required by sourcegen
+[AttributeUsage(AttributeTargets.Property)]
+public sealed class SubModuleAttribute : Attribute;
+
+[AttributeUsage(AttributeTargets.Property)]
+public sealed class WeightsAttribute : Attribute;
+
+[AttributeUsage(AttributeTargets.Property)]
+public sealed class PropertyAttribute : Attribute;
+
+[AttributeUsage(AttributeTargets.Class)]
+public sealed class GeneratedModuleAttribute(bool IncludeSerializer = false) : Attribute;
+
+[AttributeUsage(AttributeTargets.Class)]
+public sealed class GeneratedAdamAttribute(Type module) : Attribute;
+#pragma warning restore CS9113 // Parameter is unread.

ML.Core/Converters/MatrixConverter.cs

@@ -0,0 +1,32 @@
+using Ametrin.Serializer;
+
+namespace ML.Core.Converters;
+
+public sealed class MatrixConverter : ISerializationConverter<Matrix>
+{
+    static MatrixConverter()
+    {
+        AmetrinSerializer.RegisterSerializer<MatrixConverter, Matrix>();
+    }
+
+    public static Result<Matrix, DeserializationError> TryReadValue(IAmetrinReader reader)
+    {
+        using var objectReader = reader.ReadStartObject();
+        var rowCount = objectReader.ReadInt32Property("RowCount");
+        objectReader.ReadPropertyName("Storage");
+        var storage = VectorConverter.ReadValue<VectorConverter, Vector>(objectReader);
+        reader.ReadEndObject();
+        Debug.Assert(storage.Count % rowCount == 0);
+        var columnCount = storage.Count / rowCount;
+        return Matrix.Of(rowCount, columnCount, storage);
+    }
+
+    public static void WriteValue(IAmetrinWriter writer, Matrix value)
+    {
+        using var objectWriter = writer.WriteStartObject();
+        objectWriter.WriteInt32Property("RowCount", value.RowCount);
+        objectWriter.WritePropertyName("Storage");
+        VectorConverter.WriteValue(objectWriter, value.Storage);
+        writer.WriteEndObject();
+    }
+}

ML.Core/Converters/ModuleSerializer.cs

@@ -0,0 +1,49 @@
+using System.IO;
+using System.Runtime.CompilerServices;
+using Ametrin.Serializer;
+using Ametrin.Serializer.Readers;
+using Ametrin.Serializer.Writers;
+using ML.Core.Modules;
+
+namespace ML.Core.Converters;
+
+public static class ModuleSerializer
+{
+    public const string FILE_EXTENSION = ".gmw";
+    public const uint FORMAT_VERSION = 3;
+
+#pragma warning disable CA2255
+    [ModuleInitializer]
+#pragma warning restore
+    internal static void Init()
+    {
+        AmetrinSerializer.RegisterSerializer<SequenceModuleConverter<Vector>, SequenceModule<Vector>>();
+        AmetrinSerializer.RegisterSerializer<SequenceModuleConverter<Matrix>, SequenceModule<Matrix>>();
+        AmetrinSerializer.RegisterSerializer<SequenceModuleConverter<Tensor>, SequenceModule<Tensor>>();
+        AmetrinSerializer.RegisterSerializer<EmbeddedModule<int[], Vector, int>, EmbeddedModule<int[], Vector, int>>();
+    }
+
+    public static void Write(IModule module, FileInfo file)
+    {
+        using var stream = file.Create();
+        using var writer = new AmetrinBinaryWriter(stream);
+
+        writer.WriteStringProperty("$format", FILE_EXTENSION);
+        writer.WriteUInt32Property("$version", FORMAT_VERSION);
+
+        AmetrinSerializer.WriteDynamic(writer, module);
+    }
+
+    public static T Read<T>(FileInfo file)
+    {
+        using var stream = file.OpenRead();
+        using var reader = new AmetrinBinaryReader(stream);
+
+        var format = reader.ReadStringProperty("$format");
+        if (format is not FILE_EXTENSION) throw new InvalidOperationException();
+        var version = reader.ReadUInt32Property("$version");
+        if (version is not FORMAT_VERSION) throw new InvalidOperationException();
+
+        return AmetrinSerializer.TryReadDynamic<T>(reader).Or(e => e.Throw<T>());
+    }
+}

ML.Core/Converters/VectorConverter.cs

@@ -0,0 +1,21 @@
+using Ametrin.Serializer;
+
+namespace ML.Core.Converters;
+
+public sealed class VectorConverter : ISerializationConverter<Vector>
+{
+    static VectorConverter()
+    {
+        AmetrinSerializer.RegisterSerializer<VectorConverter, Vector>();
+    }
+
+    public static Result<Vector, DeserializationError> TryReadValue(IAmetrinReader reader)
+    {
+        return reader.TryReadArrayValue(static reader => reader.TryReadSingleValue()).Map(Vector.Of);
+    }
+
+    public static void WriteValue(IAmetrinWriter writer, Vector value)
+    {
+        writer.WriteArrayValue(value.AsSpan(), static (writer, v) => writer.WriteSingleValue(v));
+    }
+}

MachineLearning.Data/ITokenizer.cs → ML.Core/Data/ITokenizer.cs

@@ -1,11 +1,10 @@
-namespace MachineLearning.Data;
+namespace ML.Core.Data;
 
 public interface ITokenizer<TData>
 {
     public int TokenCount { get; }
     public IEnumerable<int> Tokenize(TData data);
     public int TokenizeSingle(TData data);
     public TData GetToken(int data);
-    public string Decode(IEnumerable<int> tokens);
-
-}
+    public TData Decode(IEnumerable<int> tokens);
+}