using ln.collections; using ln.type; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; /* * BinaryObjectFile Format * * Offset Type Description * 0x0000 byte[4] Magic Number "BOF0" * 0x0004 int32 FeatureFlags * 0x0008 long offset of first binary object * * BinaryObject * Guid UID UID of this object * int32 Version Version of this object * int32 phySize size of this binary object * int32 logSize size of used payload * int32 reserved Reserved 0 * byte[phySize] object data * */ namespace ln.objects.storage { public enum BOFFeatures : int { NONE = 0 } public class BinaryObjectFile : IDisposable { public readonly byte[] MagicNumber = Encoding.ASCII.GetBytes("BOF0"); public string FileName { get; } public BOFFeatures Features { get; private set; } = BOFFeatures.NONE; int granularity = 12; public int Granularity { get => granularity; set { if (granularity < 6) throw new ArgumentOutOfRangeException(nameof(value), "Granularity must be >= 6"); granularity = value; } } public int GranularSize => (1 << granularity); public int GranularMask => (1 << granularity) - 1; BinaryObjectCache binaryObjects; BTree unusedBinaryObjects; FileStream storageFile; long appendOffset; public BinaryObjectFile(string filename) { FileName = filename; } public void Open() { if (!File.Exists(FileName)) { Create(); } else { OpenStorage(); } } private void Create() { storageFile = new FileStream(FileName, FileMode.CreateNew, FileAccess.ReadWrite); storageFile.WriteBytes(MagicNumber); storageFile.WriteInteger((int)Features); storageFile.WriteLong(4096); storageFile.Flush(); binaryObjects = new BinaryObjectCache(); unusedBinaryObjects = new BTree(); appendOffset = 4096; } private void OpenStorage() { storageFile = new FileStream(FileName, FileMode.Open, FileAccess.ReadWrite); binaryObjects = new BinaryObjectCache(); unusedBinaryObjects = new BTree(); if (!storageFile.ReadBytes(MagicNumber.Length).AreEqual(MagicNumber)) throw new FormatException("Magic Number does not match"); Features = (BOFFeatures)storageFile.ReadInteger(); long offset = storageFile.ReadLong(); while (offset < storageFile.Length) { BinaryObject binaryObject = ReadBinaryObject(offset); if (Guid.Empty.Equals(binaryObject.UID)) unusedBinaryObjects.Add(binaryObject.Offset, binaryObject); else binaryObjects.Add(binaryObject); offset += binaryObject.PhySize; } appendOffset = offset; } private BinaryObject ReadBinaryObject(long offset) { storageFile.Position = offset; BinaryObject binaryObject = new BinaryObject(); binaryObject.UID = new Guid(storageFile.ReadBytes(16)); binaryObject.Version = storageFile.ReadInteger(); binaryObject.Offset = offset; binaryObject.PhySize = storageFile.ReadInteger(); binaryObject.LogSize = storageFile.ReadInteger(); storageFile.ReadInteger(); return binaryObject; } private void WriteBinaryObject(BinaryObject binaryObject) { storageFile.Position = binaryObject.Offset; storageFile.WriteBytes(binaryObject.UID.ToByteArray()); storageFile.WriteInteger(binaryObject.Version); storageFile.WriteInteger(binaryObject.PhySize); storageFile.WriteInteger(binaryObject.LogSize); storageFile.WriteInteger(0); } private void WriteBinaryObject(BinaryObject binaryObject,byte[] data) { if (data.Length > (binaryObject.PhySize - 32)) throw new ArgumentException(nameof(data),"more physical space needed for data"); binaryObject.LogSize = data.Length; WriteBinaryObject(binaryObject); storageFile.WriteBytes(data); } public IEnumerable ObjectUIDs => binaryObjects.UIDs; public IEnumerable GetObjectVersions(Guid uid) => binaryObjects.GetBinaryObjects(uid).Select(bo => bo.Version); public bool Contains(Guid uid) => binaryObjects.Contains(uid); public byte[] ReadBinaryObject(Guid uid) => ReadBinaryObject(uid, -1); public byte[] ReadBinaryObject(Guid uid,int version) { lock (this) { BinaryObject binaryObject = (version == -1) ? binaryObjects.GetLatestBinaryObject(uid) : binaryObjects.GetBinaryObject(uid, version); storageFile.Position = binaryObject.Offset + 32; return storageFile.ReadBytes(binaryObject.LogSize); } } public bool TryReadBinaryObject(Guid uid,out byte[] serializedBytes) => TryReadBinaryObject(uid, -1, out serializedBytes); public bool TryReadBinaryObject(Guid uid, int version, out byte[] serializedBytes) { lock (this) { if (binaryObjects.TryGetBinaryObject(uid,version, out BinaryObject binaryObject)) { storageFile.Position = binaryObject.Offset + 32; serializedBytes = storageFile.ReadBytes(binaryObject.LogSize); return true; } } serializedBytes = null; return false; } public void WriteBinaryObject(Guid uid, byte[] data) { lock (this) { if (!FindUnusedBinaryObject(data.Length + 32, out BinaryObject binaryObject)) if (!AppendBinaryObject(data.Length + 32, out binaryObject)) throw new OutOfMemoryException("could not append to storage"); if (binaryObjects.TryGetLatestBinaryObject(uid, out BinaryObject latestObject)) binaryObject.Version = latestObject.Version + 1; binaryObject.UID = uid; WriteBinaryObject(binaryObject, data); binaryObjects.Add(binaryObject); } } public void RemoveBinaryObject(Guid uid, int version) { lock (this) { BinaryObject binaryObject = binaryObjects.GetBinaryObject(uid, version); binaryObjects.Remove(binaryObject); PushUnusedBinaryObject(ref binaryObject); WriteBinaryObject(binaryObject); } } public void RemoveBinaryObjects(Guid uid) { lock (this) { foreach (int version in GetObjectVersions(uid)) { RemoveBinaryObject(uid, version); } } } private bool FindUnusedBinaryObject(int minPhySize,out BinaryObject binaryObject) { minPhySize = (minPhySize + GranularMask) & ~GranularMask; foreach (BinaryObject candidate in unusedBinaryObjects.Values) { if (candidate.PhySize >= minPhySize) { unusedBinaryObjects.Remove(candidate.Offset); binaryObject = candidate; if ((binaryObject.PhySize - minPhySize) >= GranularSize) { BinaryObject splitObject = new BinaryObject(binaryObject.Offset + minPhySize, binaryObject.PhySize - minPhySize); binaryObject.PhySize = minPhySize; WriteBinaryObject(splitObject); unusedBinaryObjects.Add(splitObject.Offset, splitObject); } return true; } } binaryObject = null; return false; } private void PushUnusedBinaryObject(ref BinaryObject binaryObject) { binaryObject.UID = Guid.Empty; unusedBinaryObjects.Add(binaryObject.Offset, binaryObject); if (unusedBinaryObjects.TryGetPreviousValue(binaryObject.Offset, out BinaryObject previousObject)) { if ((previousObject.Offset + previousObject.PhySize) == binaryObject.Offset) { previousObject.PhySize += binaryObject.PhySize; unusedBinaryObjects.Remove(binaryObject.Offset); binaryObject = previousObject; } } if (unusedBinaryObjects.TryGetNextValue(binaryObject.Offset, out BinaryObject nextObject)) { if ((binaryObject.Offset + binaryObject.PhySize) == nextObject.Offset) { binaryObject.PhySize += nextObject.PhySize; unusedBinaryObjects.Remove(nextObject.Offset); } } } private bool AppendBinaryObject(int minPhySize,out BinaryObject binaryObject) { minPhySize = (minPhySize + GranularMask) & ~GranularMask; binaryObject = new BinaryObject(appendOffset, minPhySize); appendOffset += minPhySize; return true; } public void Close() { lock (this) { storageFile.Close(); } } public void Flush() => storageFile.Flush(); public void Dispose() { storageFile?.Dispose(); storageFile = null; } } }