The best method to implement Vectors and Scalars in LLMs

OzAIVector

Putting aside all the checking and the default setting, we can see that the only instance parameters are Part (a number for how much of the vector needs to undergo RMS Norm), and the Gain. The only hyper-parameter that is shared amongst all RMS Norms is the epsilon value used. To understand what an OzAIVector is we should take a look at its code:

public abstract partial class OzAIVector
{
	public abstract OzAINumType GetNumType();
	public abstract bool GetProcMode(out OzAIProcMode res, out string error);
	
	public static bool Create(OzAIProcMode mode, out OzAIVector res, out string error)
	{
		res = null;
		if (mode == null)
		{
			error = "Could not create an OzAIVector, because no processing mode provided.";
			return false;
		}
		if (!mode.GetCPUSettings(out var cpu, out error))
		{
			error = "Could not create an OzAIVector: " + error;
			return false;
		}
		var type = cpu.DefaultProcType;
		if (!type.CreateVec(mode, out res, out error))
		{
			error = "Could not create an OzAIVector: " + error;
			return false;
		}
		return true;
	}

	public virtual bool ToDType(OzAINumType type, out OzAIVector res, out string error)
	{
		res = null;
		if (type == OzAINumType.None)
		{
			error = "Could not cast OzAIVector to specified Data Type, because no conversion to 'None' exists.";
			return false;
		}
		if (!GetProcMode(out var mode, out error))
		{
			error = "Could not cast OzAIVector to specified Data Type: " + error;
			return false;
		}
		if (!type.CreateVec(mode, out res, out error))
		{
			error = "Could not cast OzAIVector to specified Data Type: " + error;
			return false;
		}
		if (!ToFloat(out var resFloats, out error))
		{
			error = "Could not cast OzAIVector to specified Data Type: " + error;
			return false;
		}
		if (!res.Init(resFloats, 0, (ulong)resFloats.LongLength, out error))
		{
			error = "Could not cast OzAIVector to specified Data Type: " + error;
			return false;
		}
		return true;
	}
	
	// Discription
	public abstract bool GetSize(out ulong size, out string error);
	public abstract bool GetNumCount(out ulong size, out string error);
	public abstract bool GetBlockCount(out ulong size, out string error);
	public abstract bool GetBytesPerBlock(out ulong size, out string error);
	public abstract bool GetNumsPerBlock(out ulong size, out string error);
	
	// Initialization
	public abstract bool Init(ulong length, out string error);
	public abstract bool Init(byte[] data, ulong byteOffset, ulong byteCount, out string error);
	public abstract bool Init(float[] data, ulong offset, ulong length, out string error);
	public abstract bool ToBytes(out byte[] res, out string error);
	public abstract bool ToFloat(out float[] res, out string error);
	
	// Operations
	public abstract bool GetNth(ulong index, out float res, out string error);
	public abstract bool Clone(out OzAIVector res, out string error);
}

Aside form the ToDType and Create function, most of the functions are just there for utility, overridden later down the line. At its essence all OzAIVector classes are just a wrapper around an array. By making the vector class abstract, I give room for implementation where this array is not stored in RAM, but in VRAM or as quantized blocks. For example here is the main part of the OzAIHalfVec_CSharp class:

As we can see, one may replace Half with any other data type or construction as long as it answers to all the questions the abstract getters impose in the OzAIVector class.

OzAIScalar

The other type of data storage used was an OzAIScalar:

public class OzAIScalar : OzAIVectorRange
{
	public static bool Create(float val, OzAIProcMode mode, out OzAIScalar res, out string error)
	{
		res = new OzAIScalar();
		res.Offset = 0;
		if (!OzAIVector.Create(mode, out res.Vector, out error))
		return false;
		if (!res.Vector.Init([val], 0, 1, out error))
		return false;
		return true;
	}

	public static bool CreateInt(int val, OzAIProcMode mode, out OzAIScalar res, out string error)
	{
		res = new OzAIScalar();
		res.Offset = 0;
		if (!OzAIIntVec.Create(mode, out var vec, out error))
		return false;
		res.Vector = vec;
		if (!res.Vector.Init(1, out error))
		return false;
		if (!vec.SetNthInt(val, 0, out error))
		return false;
		return true;
	}

	public static bool CreateFloat(float val, OzAIProcMode mode, out OzAIScalar res, out string error)
	{
		res = new OzAIScalar();
		res.Offset = 0;
		if (!OzAIFloatVec.Create(mode, out var vec, out error))
		return false;
		res.Vector = vec;
		if (!res.Vector.Init([val], 0, 1, out error))
		return false;
		return true;
	}

	public static bool CreateHalf(Half val, OzAIProcMode mode, out OzAIScalar res, out string error)
	{
		res = new OzAIScalar();
		res.Offset = 0;
		if (!OzAIHalfVec.Create(mode, out var vec, out error))
		return false;
		res.Vector = vec;
		if (!res.Vector.Init(1, out error))
		return false;
		if (!vec.SetNthHalf(val, 0, out error))
		return false;
		return true;
	}

	public override ulong Length { get => 1; set { } }
} 

It should come as no surprise that it is mere a vector with a length one. The reason why this is powerful is because the number it is pointing to can be represented in any format supported by a vector. Matrices are not presented but behave similarly to the vector class. The elements are stored in one flat array in row major order.

OzAIMemNode

Finally, returning to the topic of the RMS Norm component, one should note that it used the OzAICompIOMem_Unary class to access I/O data. This class is as follows:

public class OzAICompIOMem_Unary : OzAICompIOMem
{
	public OzAIMemNode Inputs;
	public OzAIMemNode Outputs;
	
	public override bool IsPossible(out string error)
	{
		return CheckIfNull([Inputs, Outputs], ["Inputs", "Outputs"], out error);
	}
}

Here, a memory node is just a wrapper around a List<OzAIVector> which contains some useful functions. The key is that this allows for not only a deep but also a shallow copy of the list of vectors/vector, thereby greatly reducing the actual amount of data copied. Here is a snippet of this class:

public class OzAIMemNode
{
	public OzAIMemNode()
	{
		_vecs = new List<OzAIVector>();
	}
	
	List<OzAIVector> _vecs;
	
	public ulong Count
	{
		get
		{
			return (ulong)_vecs.LongCount();
		}
	}

	public void Add(OzAIVector vec)
	{
		_vecs.Add(vec);
	}
	public void Add(OzAIVector[] vecs)
	{
		_vecs.AddRange(vecs);
	}
	public void Add(List<OzAIVector> vecs)
	{
		_vecs.AddRange(vecs);
	}
	
	…
	
	public bool AddVecs(OzAIProcMode mode, ulong len, ulong count, out string error)
	{
		for (ulong i = 0; i < count; i++)
		{
			if (!OzAIVector.Create(mode, out var vec, out error))
			{
				error = "Could not create vector in memory node: " + error;
				return false;
			}
			if (!vec.Init(len, out error))
			{
				error = "Could not initialize vector in memory node: " + error;
				return false;
			}
			_vecs.Add(vec);
		}
		error = null;
		return true;
	}

	public List<OzAIVector> GetList()
	{
		return _vecs;
	}

	public OzAIVector[] GetArray()
	{
		return _vecs.ToArray();
	}
}