ln.objects/storage/BinaryObjectFile.cs

using ln.collections;
using ln.type;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;

/*
 * BinaryObjectFile Format
 * 
 * Offset   Type        Description
 *  0x0000  byte[4]     Magic Number "BOF0"
 *  0x0004  int32       FeatureFlags
 *  0x0008  long        offset of first binary object
 * 
 * BinaryObject
 *  Guid    UID         UID of this object
 *  int32   Version     Version of this object
 *  int32   phySize     size of this binary object 
 *  int32   logSize     size of used payload
 *  int32   reserved    Reserved 0
 *  byte[phySize]       object data
 * 
 */


namespace ln.objects.storage
{
    public enum BOFFeatures : int 
    {
        NONE = 0
    }
    public class BinaryObjectFile : IDisposable
    {
        public readonly byte[] MagicNumber = Encoding.ASCII.GetBytes("BOF0");

        public string FileName { get; }
        public BOFFeatures Features { get; private set; } = BOFFeatures.NONE;

        int granularity = 12;
        public int Granularity
        {
            get => granularity;
            set
            {
                if (granularity < 6)
                    throw new ArgumentOutOfRangeException(nameof(value), "Granularity must be >= 6");
                granularity = value;
            }
        }
        public int GranularSize => (1 << granularity);
        public int GranularMask => (1 << granularity) - 1;

        BinaryObjectCache binaryObjects;
        BTree<long, BinaryObject> unusedBinaryObjects;
        FileStream storageFile;

        long appendOffset;

        public BinaryObjectFile(string filename)
        {
            FileName = filename;
        }

        public void Open()
        {
            if (!File.Exists(FileName))
            {
                Create();
            } else
            {
                OpenStorage();
            }
        }
        private void Create()
        {
            storageFile = new FileStream(FileName, FileMode.CreateNew, FileAccess.ReadWrite);
            storageFile.WriteBytes(MagicNumber);
            storageFile.WriteInteger((int)Features);
            storageFile.WriteLong(4096);
            storageFile.Flush();

            binaryObjects = new BinaryObjectCache();
            unusedBinaryObjects = new BTree<long, BinaryObject>();

            appendOffset = 4096;
        }
        private void OpenStorage()
        {
            storageFile = new FileStream(FileName, FileMode.Open, FileAccess.ReadWrite);
            binaryObjects = new BinaryObjectCache();
            unusedBinaryObjects = new BTree<long, BinaryObject>();

            if (!storageFile.ReadBytes(MagicNumber.Length).AreEqual(MagicNumber))
                throw new FormatException("Magic Number does not match");

            Features = (BOFFeatures)storageFile.ReadInteger();

            long offset = storageFile.ReadLong();
            
            while (offset < storageFile.Length)
            {
                BinaryObject binaryObject = ReadBinaryObject(offset);
                if (Guid.Empty.Equals(binaryObject.UID))
                    unusedBinaryObjects.Add(binaryObject.Offset, binaryObject);
                else
                    binaryObjects.Add(binaryObject);

                offset += binaryObject.PhySize;
            }
            
            appendOffset = offset;
        }

        private BinaryObject ReadBinaryObject(long offset)
        {
            storageFile.Position = offset;

            BinaryObject binaryObject = new BinaryObject();
            
            binaryObject.UID = new Guid(storageFile.ReadBytes(16));
            binaryObject.Version = storageFile.ReadInteger();
            binaryObject.Offset = offset;
            binaryObject.PhySize = storageFile.ReadInteger();
            binaryObject.LogSize = storageFile.ReadInteger();
            storageFile.ReadInteger();

            return binaryObject;
        }
        private void WriteBinaryObject(BinaryObject binaryObject)
        {
            storageFile.Position = binaryObject.Offset;

            storageFile.WriteBytes(binaryObject.UID.ToByteArray());
            storageFile.WriteInteger(binaryObject.Version);
            storageFile.WriteInteger(binaryObject.PhySize);
            storageFile.WriteInteger(binaryObject.LogSize);
            storageFile.WriteInteger(0);
        }
        private void WriteBinaryObject(BinaryObject binaryObject,byte[] data)
        {
            if (data.Length > (binaryObject.PhySize - 32))
                throw new ArgumentException(nameof(data),"more physical space needed for data");

            binaryObject.LogSize = data.Length;
            WriteBinaryObject(binaryObject);
            storageFile.WriteBytes(data);
        }


        public IEnumerable<Guid> ObjectUIDs => binaryObjects.UIDs;
        public IEnumerable<int> GetObjectVersions(Guid uid) => binaryObjects.GetBinaryObjects(uid).Select(bo => bo.Version);
        public bool Contains(Guid uid) => binaryObjects.Contains(uid);

        public byte[] ReadBinaryObject(Guid uid) => ReadBinaryObject(uid, -1);
        public byte[] ReadBinaryObject(Guid uid,int version)
        {
            lock (this)
            {
                BinaryObject binaryObject = (version == -1) ? binaryObjects.GetLatestBinaryObject(uid) : binaryObjects.GetBinaryObject(uid, version);
                storageFile.Position = binaryObject.Offset + 32;
                return storageFile.ReadBytes(binaryObject.LogSize);
            }
        }
        public bool TryReadBinaryObject(Guid uid,out byte[] serializedBytes) => TryReadBinaryObject(uid, -1, out serializedBytes);
        public bool TryReadBinaryObject(Guid uid, int version, out byte[] serializedBytes)
        {
            lock (this)
            {
                if (binaryObjects.TryGetBinaryObject(uid,version, out BinaryObject binaryObject))
                {
                    storageFile.Position = binaryObject.Offset + 32;
                    serializedBytes = storageFile.ReadBytes(binaryObject.LogSize);
                    return true;
                }
            }
            serializedBytes = null;
            return false;
        }

        public void WriteBinaryObject(Guid uid, byte[] data)
        {
            lock (this)
            {
                if (!FindUnusedBinaryObject(data.Length + 32, out BinaryObject binaryObject))
                    if (!AppendBinaryObject(data.Length + 32, out binaryObject))
                        throw new OutOfMemoryException("could not append to storage");

                if (binaryObjects.TryGetLatestBinaryObject(uid, out BinaryObject latestObject))
                    binaryObject.Version = latestObject.Version + 1;

                binaryObject.UID = uid;
                WriteBinaryObject(binaryObject, data);
                binaryObjects.Add(binaryObject);
            }
        }

        public void RemoveBinaryObject(Guid uid, int version)
        {
            lock (this)
            {
                BinaryObject binaryObject = binaryObjects.GetBinaryObject(uid, version);
                binaryObjects.Remove(binaryObject);
                PushUnusedBinaryObject(ref binaryObject);
                WriteBinaryObject(binaryObject);
            }
        }
        public void RemoveBinaryObjects(Guid uid)
        {
            lock (this)
            {
                foreach (int version in GetObjectVersions(uid))
                {
                    RemoveBinaryObject(uid, version);
                }
            }
        }

        private bool FindUnusedBinaryObject(int minPhySize,out BinaryObject binaryObject)
        {
            minPhySize = (minPhySize + GranularMask) & ~GranularMask;

            foreach (BinaryObject candidate in unusedBinaryObjects.Values)
            {
                if (candidate.PhySize >= minPhySize)
                {
                    unusedBinaryObjects.Remove(candidate.Offset);

                    binaryObject = candidate;

                    if ((binaryObject.PhySize - minPhySize) >= GranularSize)
                    {
                        BinaryObject splitObject = new BinaryObject(binaryObject.Offset + minPhySize, binaryObject.PhySize - minPhySize);
                        binaryObject.PhySize = minPhySize;
                        WriteBinaryObject(splitObject);
                        unusedBinaryObjects.Add(splitObject.Offset, splitObject);
                    }

                    return true;
                }
            }
            binaryObject = null;
            return false;
        }

        private void PushUnusedBinaryObject(ref BinaryObject binaryObject)
        {
            binaryObject.UID = Guid.Empty;
            unusedBinaryObjects.Add(binaryObject.Offset, binaryObject);

            if (unusedBinaryObjects.TryGetPreviousValue(binaryObject.Offset, out BinaryObject previousObject))
            {
                if ((previousObject.Offset + previousObject.PhySize) == binaryObject.Offset)
                {
                    previousObject.PhySize += binaryObject.PhySize;
                    unusedBinaryObjects.Remove(binaryObject.Offset);
                    binaryObject = previousObject;
                }
            }
            if (unusedBinaryObjects.TryGetNextValue(binaryObject.Offset, out BinaryObject nextObject))
            {
                if ((binaryObject.Offset + binaryObject.PhySize) == nextObject.Offset)
                {
                    binaryObject.PhySize += nextObject.PhySize;
                    unusedBinaryObjects.Remove(nextObject.Offset);
                }
            }
        }

        private bool AppendBinaryObject(int minPhySize,out BinaryObject binaryObject)
        {
            minPhySize = (minPhySize + GranularMask) & ~GranularMask;
            binaryObject = new BinaryObject(appendOffset, minPhySize);
            appendOffset += minPhySize;
            return true;
        }

        public void Close()
        {
            lock (this)
            {
                storageFile.Close();
            }
        }

        public void Flush() => storageFile.Flush();


        public void Dispose()
        {
            storageFile?.Dispose();
            storageFile = null;
        }
    }
}