ln.bson/ln.bson.storage/ByteArrayFileStorage.cs

using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Xml;
using ln.type;

namespace ln.bson.storage
{
    public class ByteArrayFileStorage : IDisposable, IEnumerable<KeyValuePair<long,byte[]>>
    {
        private string _baseFileName;
        private string _offsetFileName;
        private FileStream _fileStream;
        private SortedSet<long> _storageOffsets = new SortedSet<long>();
        
        public Guid CurrentGuid { get; private set; }

        public ByteArrayFileStorage(string baseFilename) : this(baseFilename, baseFilename + ".offset")
        {
        }

        public ByteArrayFileStorage(string baseFilename, string offsetFileName)
        {
            _baseFileName = baseFilename;
            _offsetFileName = offsetFileName;
            
            _fileStream = new FileStream(baseFilename, FileMode.OpenOrCreate, FileAccess.ReadWrite);
            _fileStream.Position = 0;
            if (_fileStream.Length == 0)
            {
                CurrentGuid = Guid.NewGuid();
                _fileStream.Write(CurrentGuid.ToByteArray());
            }
            else
            {
                CurrentGuid = new Guid(_fileStream.ReadBytes(16));
            }
            
            if (File.Exists(_offsetFileName))
            {
                using (FileStream _offsetStream = new FileStream(offsetFileName, FileMode.Open, FileAccess.Read))
                {
                    Guid offsetGuid = new Guid(_offsetStream.ReadBytes(16));
                    if (offsetGuid.Equals(CurrentGuid))
                    {
                        byte[] offsets = _offsetStream.ReadToEnd();
                        if ((offsets.Length % 8) != 0)
                            throw new InvalidDataException("offsetfile has invalid length");
                        
                        int n = 0;
                        while (n < offsets.Length)
                        {
                            _storageOffsets.Add(BitConverter.ToInt64(offsets, n));
                            n += 8;
                        }
                    }
                }
            }
            
            if (_storageOffsets.Count == 0)
                ScanOffsets();
            
            _fileStream.Position = 0;
            _fileStream.WriteBytes(Guid.Empty.ToByteArray());
            _fileStream.Flush();
        }

        void ScanOffsets()
        {
            long currentStorageOffset = 16;
            int currentStorageSize = 0;

            while (currentStorageOffset < _fileStream.Length)
            {
                _fileStream.Position = currentStorageOffset;
                currentStorageSize = _fileStream.ReadInteger();

                if (currentStorageSize < 0)
                {
                    _storageOffsets.Add(-currentStorageOffset);
                    currentStorageOffset -= currentStorageSize;
                }
                else
                {
                    _storageOffsets.Add(currentStorageOffset);
                    currentStorageOffset += currentStorageSize;   
                }

                currentStorageOffset += 4;
            }
        }

        void SaveOffsets()
        {
            byte[] offsets = new byte[_storageOffsets.Count * 8];
            int n = 0;
            foreach (long storageOffset in _storageOffsets)
            {
                Array.Copy(BitConverter.GetBytes(storageOffset), 0, offsets, n * 8, 8);
                n++;
            }

            using (FileStream _offsetStream = new FileStream(_offsetFileName, FileMode.Create, FileAccess.Write))
            {
                _offsetStream.WriteBytes(CurrentGuid.ToByteArray());
                _offsetStream.WriteBytes(offsets);
            }
        }

        public bool TryLoadDocumentBytes(long documentOffset, out byte[] documentBytes)
        {
            lock (this)
            {
                if (!_storageOffsets.Contains(documentOffset))
                    throw new IOException();

                _fileStream.Position = documentOffset;
                int storageSize = _fileStream.ReadInteger();
                if (storageSize < 0)
                {
                    documentBytes = null;
                    return false;
                }
                else
                {
                    int documentSize = _fileStream.ReadInteger();
                    documentBytes = new byte[documentSize];
                    int nRead = _fileStream.Read(documentBytes, 0, documentSize);
                    if (nRead != documentSize)
                        throw new IOException();
                    return true;
                }
            }
        }
        
        public bool TrySaveDocumentBytes(byte[] documentBytes, out long documentOffset)
        {
            lock (this)
            {
                foreach (long currentDocumentOffset in _storageOffsets.GetViewBetween(long.MinValue, 0).Reverse())
                {
                    _fileStream.Position = -currentDocumentOffset;
                    int currentStorageSize = -_fileStream.ReadInteger();
                    if (currentStorageSize >= documentBytes.Length + 4)
                    {
                        _fileStream.Position = -currentDocumentOffset;
                        _fileStream.WriteInteger(
                            currentStorageSize > (documentBytes.Length + 16)
                            ? documentBytes.Length + 4
                            : currentStorageSize
                            );
                        documentOffset = -currentDocumentOffset;
                        _fileStream.WriteInteger(documentBytes.Length);
                        _fileStream.Write(documentBytes, 0, documentBytes.Length);
                        CurrentGuid = Guid.NewGuid();
                        _storageOffsets.Remove(currentDocumentOffset);
                        _storageOffsets.Add(documentOffset);

                        if (currentStorageSize > (documentBytes.Length + 16))
                        {
                            long nextOffset = documentOffset + documentBytes.Length + 4;
                            int nextSize = currentStorageSize - documentBytes.Length - 8;
                            _fileStream.Position = nextOffset;
                            _fileStream.WriteInteger(-nextSize);
                            long testOffset = nextOffset + nextSize + 4;
                            if ((testOffset != _fileStream.Length) && 
                                !_storageOffsets.Contains(-testOffset) &&
                                !_storageOffsets.Contains(testOffset))
                                throw new IOException("testOffset failed");
                            _storageOffsets.Add(-nextOffset);
                        }
                        
                        _fileStream.Flush();
                        return true;
                    }
                }

                documentOffset = _fileStream.Length;
                _fileStream.Position = _fileStream.Length;
                _fileStream.WriteInteger(documentBytes.Length + 4);
                _fileStream.WriteInteger(documentBytes.Length);
                _fileStream.Write(documentBytes, 0, documentBytes.Length);
                CurrentGuid = Guid.NewGuid();
                _fileStream.Flush();
                _storageOffsets.Add(documentOffset);
                return true;
            }
        }

        public bool Remove(long documentOffset)
        {
            lock (this)
            {
                if (!_storageOffsets.Contains(documentOffset))
                    throw new IOException();
                
                _fileStream.Position = documentOffset;
                int currentStorageSize = _fileStream.ReadInteger();
                if (currentStorageSize > 0)
                {
                    currentStorageSize = -currentStorageSize;
                    _fileStream.Position = documentOffset;
                    _fileStream.WriteInteger(currentStorageSize);
                    CurrentGuid = Guid.NewGuid();
                    _storageOffsets.Remove(documentOffset);
                    _storageOffsets.Add(-documentOffset);
                    return true;
                }
            }

            return false;
        }

        public IEnumerable<long> Offsets => _storageOffsets.GetViewBetween(0,long.MaxValue);


        public void Close()
        {
            if (_fileStream == null)
                throw new ObjectDisposedException(nameof(ByteArrayFileStorage));
            
            SaveOffsets();
            _fileStream.Position = 0;
            _fileStream.WriteBytes(CurrentGuid.ToByteArray());
            _fileStream.Flush();
            _fileStream.Close();
            _fileStream = null;
        }
        
        
        class SimpleDocumentOffsetEnumerator : IEnumerator<KeyValuePair<long,byte[]>>
        {
            private ByteArrayFileStorage _fileStorage;
            private IEnumerator<long> _offsetEnumerator;
            private byte[] currentDocument;

            public SimpleDocumentOffsetEnumerator(ByteArrayFileStorage fileStorage)
            {
                this._fileStorage = fileStorage;
            }
            
            public bool MoveNext()
            {
                if (_offsetEnumerator is null)
                    _offsetEnumerator = _fileStorage.Offsets.GetEnumerator();

                if (_offsetEnumerator.MoveNext())
                {
                    if (_fileStorage.TryLoadDocumentBytes(_offsetEnumerator.Current, out currentDocument))
                        return true;
                }
                return false;
            }

            public void Reset()
            {
                _offsetEnumerator?.Dispose();
                _offsetEnumerator = null;
                currentDocument = null;
            }

            public KeyValuePair<long, byte[]> Current =>
                new KeyValuePair<long, byte[]>(_offsetEnumerator.Current, currentDocument);

            object IEnumerator.Current => Current;

            public void Dispose()
            {
                currentDocument = null;
            }
        }

/*
        class SimpleDocumentEnumerator : IEnumerator<BsonDocument>
        {
            private SimpleDocumentOffsetEnumerator _offsetEnumerator;
            
            public SimpleDocumentEnumerator(ByteArrayFileStorage fileStorage)
            {
                _offsetEnumerator = new SimpleDocumentOffsetEnumerator(fileStorage);
            }

            public bool MoveNext() => _offsetEnumerator.MoveNext();
            public void Reset() => _offsetEnumerator.Reset();

            public BsonDocument Current => _offsetEnumerator.Current.Value;
            object IEnumerator.Current => Current;

            public void Dispose()
            {
                _offsetEnumerator?.Dispose();
            }
        }
        */

        public void Dispose()
        {
            if (_fileStream != null)
                Close();
            
            _fileStream?.Dispose();
            _fileStream = null;
        }

        public IEnumerator<KeyValuePair<long, byte[]>> GetEnumerator() => new SimpleDocumentOffsetEnumerator(this);
        IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
    }
    
    /*
bool TryGetCachedDocument(long documentOffset, out BsonDocument bsonDocument)
{
    if (_documentsCache.TryGetValue(documentOffset, out WeakReference<BsonDocument> weakDocument))
    {
        if (weakDocument.TryGetTarget(out bsonDocument))
            return true;

        _documentsCache.Remove(documentOffset);
    }

    bsonDocument = null;
    return false;
}

public bool TryLoadDocument(long documentOffset, out BsonDocument bsonDocument)
{
    lock (this)
    {
        if (!_documentOffsets.Contains(documentOffset))
            throw new IOException();
        
        if (TryGetCachedDocument(documentOffset, out bsonDocument))
            return true;
        
        if (TryLoadDocumentBytes(documentOffset, out int documentSize, out byte[] documentBytes))
        {
            bsonDocument = BsonReader.ReadDocument(documentBytes, 0, documentSize);
            bsonDocument.StorageTag = documentOffset;
            _documentsCache.Add(documentOffset, new WeakReference<BsonDocument>(bsonDocument));
            return true;
        }
    }

    return false;
}
*/

}