using System; using System.Collections; using System.Collections.Generic; using System.IO; using ln.bson.storage.index; using ln.collections; using ln.type; namespace ln.bson.storage { public class BsonDocumentStorageConfiguration { private bool _sealed; public string BaseFileName { get; } private Dictionary> indeces = new Dictionary>(); public IEnumerable>> Indeces => indeces; private Action _logTarget; public Action LogTarget => _logTarget; public BsonDocumentStorageConfiguration(string baseFileName) { BaseFileName = baseFileName; _logTarget = (s) => { }; } public BsonDocumentStorageConfiguration Index(string path) => Index(path, BsonIndex.CompareBytesLittleEndian); public BsonDocumentStorageConfiguration Index(string path, Comparison comparison) { if (_sealed) throw new ApplicationException(); indeces.Add(path, comparison); return this; } public BsonDocumentStorageConfiguration SetLogTarget(Action logTarget) { if (_sealed) throw new ApplicationException(); _logTarget = logTarget; return this; } internal BsonDocumentStorageConfiguration Seal() { _sealed = true; return this; } } public class BsonDocumentStorage : IEnumerable, IDisposable { private BsonDocumentStorageConfiguration _configuration; private ByteArrayFileStorage _fileStorage; private WeakValueDictionary _documentCache = new WeakValueDictionary(); private WeakKeyDictionary _reverseCache = new WeakKeyDictionary(); private IndexTree _indexTree = new IndexTree(); public BsonDocumentStorageConfiguration Configuration => _configuration; public BsonDocumentStorage(string baseFileName) : this(new BsonDocumentStorageConfiguration(baseFileName)) { } public BsonDocumentStorage(BsonDocumentStorageConfiguration configuration) { _configuration = configuration.Seal(); _fileStorage = new ByteArrayFileStorage(_configuration.BaseFileName); SetupIndeces(); } public void Close() { lock (this) { SaveIndeces(); _fileStorage.Close(); } } public BsonDocument LoadDocument(long documentOffset) { if (TryLoadDocument(documentOffset, out BsonDocument bsonDocument)) return bsonDocument; throw new KeyNotFoundException(); } public bool TryLoadDocument(long documentOffset, out BsonDocument bsonDocument) { lock (this) { if (TryGetCachedDocument(documentOffset, out bsonDocument)) return true; if (_fileStorage.TryLoadDocumentBytes(documentOffset, out byte[] documentBytes)) { bsonDocument = BsonReader.ReadDocument(documentBytes); TryAddOrUpdateCachedDocument(documentOffset, bsonDocument); return true; } } return false; } /** * Save Document to Storage * If bsonDocument originated from this store, replaces this document * If bsonDocument is unknown to this store, the document is added */ public bool SaveDocument(BsonDocument bsonDocument) { lock (this) { TryGetCachedDocumentOffset(bsonDocument, out long documentOffset); byte[] documentBytes = bsonDocument.GetBytes(); if (_fileStorage.TrySaveDocumentBytes(documentBytes, out long newDocumentOffset)) { if (documentOffset != 0) _documentCache.Remove(documentOffset); _reverseCache[bsonDocument] = newDocumentOffset; _documentCache.Add(newDocumentOffset, bsonDocument); if (documentOffset != 0) _indexTree.ReplaceDocument(documentOffset, newDocumentOffset, bsonDocument); else _indexTree.AddDocument(newDocumentOffset, bsonDocument); return true; } } return false; } /** * Remove bsonDocument from the storage * Returns true if document has been successfully removed * Returns false if document is not known to storage or can't be removed successfully */ public bool DeleteDocument(BsonDocument bsonDocument) { lock (this) { if (TryGetCachedDocumentOffset(bsonDocument, out long documentOffset) && _fileStorage.Remove(documentOffset)) { _documentCache.Remove(documentOffset); _reverseCache.Remove(bsonDocument); _indexTree.RemoveDocument(documentOffset); return true; } } return false; } /* Document Cache */ public bool TryGetCachedDocument(long documentOffset, out BsonDocument cachedDocument) => _documentCache.TryGetValue(documentOffset, out cachedDocument); public bool TryGetCachedDocumentOffset(BsonDocument bsonDocument, out long documentOffset) => _reverseCache.TryGetValue(bsonDocument, out documentOffset); public bool TryAddOrUpdateCachedDocument(long documentOffset, BsonDocument bsonDocument) { if (_documentCache.TryGetValue(documentOffset, out BsonDocument cachedDocument)) { _reverseCache.Remove(cachedDocument); _documentCache.Remove(documentOffset); } _documentCache.Add(documentOffset, bsonDocument); _reverseCache.Add(bsonDocument, documentOffset); return true; } /* End Document Cache */ /* Index methods */ private void SetupIndeces() { bool rebuildNeeded = false; foreach (var indexConfiguration in _configuration.Indeces) { if (!_indexTree.EnsureIndex(indexConfiguration.Key, indexConfiguration.Value, out IBsonIndex bsonIndex)) throw new ArgumentOutOfRangeException("duplicate index path"); string indexFileName = String.Format("{0}.{1}.idx", _configuration.BaseFileName, indexConfiguration.Key); if (rebuildNeeded || !File.Exists(indexFileName) || !LoadIndexFile(indexFileName, bsonIndex)) rebuildNeeded = true; } if (rebuildNeeded) RebuildIndeces(); } void RebuildIndeces() { _configuration.LogTarget("rebuilding indeces"); _indexTree.Clear(); foreach (BsonDocument bsonDocument in this) { if (!_reverseCache.TryGetValue(bsonDocument, out long documentOffset)) throw new ApplicationException("serious bug! no document offset"); _indexTree.AddDocument(documentOffset, bsonDocument); } } private void UpdateIndex(BsonDocument bsonDocument) { if (TryGetCachedDocumentOffset(bsonDocument, out long documentOffset)) UpdateIndex(bsonDocument, documentOffset); else throw new KeyNotFoundException(); } private void UpdateIndex(BsonDocument bsonDocument, long documentOffset) { _indexTree.AddDocument(documentOffset, bsonDocument); } public IEnumerable Query(String path, QueryOperator op, BsonValue value) { HashSet resultsSet = new HashSet(); _indexTree.Query(path, op, value.GetBytes(), resultsSet); return new BsonDocumentEnumerable(() => new BsonDocumentEnumerator(this, resultsSet.GetEnumerator())); } private void SaveIndeces() { _configuration.LogTarget("saving indeces"); foreach (var indexConfiguration in _configuration.Indeces) { string indexFileName = String.Format("{0}.{1}.idx", _configuration.BaseFileName, indexConfiguration.Key); if (_indexTree.EnsureIndex(indexConfiguration.Key, indexConfiguration.Value, out IBsonIndex bsonIndex)) throw new ArgumentOutOfRangeException("unknown index path"); if (!SaveIndexFile(indexFileName, bsonIndex)) throw new Exception(String.Format("Failed to save index {0} to file {1}", indexConfiguration.Key, indexFileName)); } } bool LoadIndexFile(string indexFileName, IBsonIndex index) { try { using (FileStream indexFile = new FileStream(indexFileName, FileMode.Open, FileAccess.Read)) { Guid indexGuid = new Guid(indexFile.ReadBytes(16)); if (!indexGuid.Equals(_fileStorage.CurrentGuid)) return false; while (indexFile.Position < indexFile.Length) { long o = indexFile.ReadLong(); int l = indexFile.ReadInteger(); byte[] v = indexFile.ReadBytes(l); index.AddValue(o, v); } } return true; } catch (Exception) { return false; } } bool SaveIndexFile(string indexFileName, IBsonIndex index) { using (FileStream indexFile = new FileStream(indexFileName, FileMode.Create, FileAccess.Write)) { indexFile.WriteBytes(_fileStorage.CurrentGuid.ToByteArray()); foreach (var ie in index) { indexFile.WriteLong(ie.Key); indexFile.WriteInteger(ie.Value.Length); indexFile.WriteBytes(ie.Value); } indexFile.Flush(); indexFile.Close(); } return true; } /* End Index methods */ public void Dispose() { Close(); _fileStorage?.Dispose(); _fileStorage = null; } public class BsonDocumentEnumerable : IEnumerable { private Func> _getenumerator; public BsonDocumentEnumerable(Func> getenumerator) { _getenumerator = getenumerator; } public IEnumerator GetEnumerator() => _getenumerator(); IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); } public IEnumerator GetEnumerator() => new BsonDocumentEnumerator(this); IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); class BsonDocumentEnumerator : IEnumerator { private BsonDocumentStorage _documentStorage; private IEnumerator OffsetEnumerator; public BsonDocumentEnumerator(BsonDocumentStorage documentStorage) { _documentStorage = documentStorage; OffsetEnumerator = _documentStorage._fileStorage.Offsets.GetEnumerator(); } public BsonDocumentEnumerator(BsonDocumentStorage documentStorage, IEnumerator offsetEnumerator) { _documentStorage = documentStorage; OffsetEnumerator = offsetEnumerator; } public bool MoveNext() => OffsetEnumerator.MoveNext(); public void Reset() => OffsetEnumerator.Reset(); public BsonDocument Current => _documentStorage.LoadDocument(OffsetEnumerator.Current); object IEnumerator.Current => Current; public void Dispose() { OffsetEnumerator?.Dispose(); OffsetEnumerator = null; } } } }