361 lines
13 KiB
C#
361 lines
13 KiB
C#
using System;
|
|
using System.Collections;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using ln.bson.storage.index;
|
|
using ln.collections;
|
|
using ln.type;
|
|
|
|
namespace ln.bson.storage
|
|
{
|
|
public class BsonDocumentStorageConfiguration
|
|
{
|
|
private bool _sealed;
|
|
public string BaseFileName { get; }
|
|
|
|
private Dictionary<string, Comparison<byte[]>> indeces = new Dictionary<string, Comparison<byte[]>>();
|
|
public IEnumerable<KeyValuePair<string, Comparison<byte[]>>> Indeces => indeces;
|
|
|
|
private Action<string> _logTarget;
|
|
public Action<string> LogTarget => _logTarget;
|
|
|
|
public BsonDocumentStorageConfiguration(string baseFileName)
|
|
{
|
|
BaseFileName = baseFileName;
|
|
_logTarget = (s) => { };
|
|
}
|
|
|
|
public BsonDocumentStorageConfiguration Index(string path) => Index(path, BsonIndex.CompareBytesLittleEndian);
|
|
public BsonDocumentStorageConfiguration Index(string path, Comparison<byte[]> comparison)
|
|
{
|
|
if (_sealed)
|
|
throw new ApplicationException();
|
|
|
|
indeces.Add(path, comparison);
|
|
return this;
|
|
}
|
|
|
|
public BsonDocumentStorageConfiguration SetLogTarget(Action<string> logTarget)
|
|
{
|
|
if (_sealed)
|
|
throw new ApplicationException();
|
|
_logTarget = logTarget;
|
|
return this;
|
|
}
|
|
|
|
internal BsonDocumentStorageConfiguration Seal()
|
|
{
|
|
_sealed = true;
|
|
return this;
|
|
}
|
|
}
|
|
|
|
public class BsonDocumentStorage : IEnumerable<BsonDocument>, IDisposable
|
|
{
|
|
private BsonDocumentStorageConfiguration _configuration;
|
|
private ByteArrayFileStorage _fileStorage;
|
|
|
|
private WeakValueDictionary<long, BsonDocument> _documentCache = new WeakValueDictionary<long, BsonDocument>();
|
|
private WeakKeyDictionary<BsonDocument, long> _reverseCache = new WeakKeyDictionary<BsonDocument, long>();
|
|
|
|
private IndexTree _indexTree = new IndexTree();
|
|
|
|
public BsonDocumentStorageConfiguration Configuration => _configuration;
|
|
|
|
public BsonDocumentStorage(string baseFileName) : this(new BsonDocumentStorageConfiguration(baseFileName))
|
|
{
|
|
}
|
|
|
|
public BsonDocumentStorage(BsonDocumentStorageConfiguration configuration)
|
|
{
|
|
_configuration = configuration.Seal();
|
|
_fileStorage = new ByteArrayFileStorage(_configuration.BaseFileName);
|
|
|
|
SetupIndeces();
|
|
}
|
|
|
|
public void Close()
|
|
{
|
|
lock (this)
|
|
{
|
|
SaveIndeces();
|
|
_fileStorage.Close();
|
|
}
|
|
}
|
|
|
|
public BsonDocument LoadDocument(long documentOffset)
|
|
{
|
|
if (TryLoadDocument(documentOffset, out BsonDocument bsonDocument))
|
|
return bsonDocument;
|
|
throw new KeyNotFoundException();
|
|
}
|
|
public bool TryLoadDocument(long documentOffset, out BsonDocument bsonDocument)
|
|
{
|
|
lock (this)
|
|
{
|
|
if (TryGetCachedDocument(documentOffset, out bsonDocument))
|
|
return true;
|
|
|
|
if (_fileStorage.TryLoadDocumentBytes(documentOffset, out byte[] documentBytes))
|
|
{
|
|
bsonDocument = BsonReader.ReadDocument(documentBytes);
|
|
TryAddOrUpdateCachedDocument(documentOffset, bsonDocument);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Save Document to Storage
|
|
* If bsonDocument originated from this store, replaces this document
|
|
* If bsonDocument is unknown to this store, the document is added
|
|
*/
|
|
public bool SaveDocument(BsonDocument bsonDocument)
|
|
{
|
|
lock (this)
|
|
{
|
|
TryGetCachedDocumentOffset(bsonDocument, out long documentOffset);
|
|
byte[] documentBytes = bsonDocument.GetBytes();
|
|
if (_fileStorage.TrySaveDocumentBytes(documentBytes, out long newDocumentOffset))
|
|
{
|
|
if (documentOffset != 0)
|
|
_documentCache.Remove(documentOffset);
|
|
|
|
_reverseCache[bsonDocument] = newDocumentOffset;
|
|
_documentCache.Add(newDocumentOffset, bsonDocument);
|
|
|
|
if (documentOffset != 0)
|
|
_indexTree.ReplaceDocument(documentOffset, newDocumentOffset, bsonDocument);
|
|
else
|
|
_indexTree.AddDocument(newDocumentOffset, bsonDocument);
|
|
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Remove bsonDocument from the storage
|
|
* Returns true if document has been successfully removed
|
|
* Returns false if document is not known to storage or can't be removed successfully
|
|
*/
|
|
public bool DeleteDocument(BsonDocument bsonDocument)
|
|
{
|
|
lock (this)
|
|
{
|
|
if (TryGetCachedDocumentOffset(bsonDocument, out long documentOffset) &&
|
|
_fileStorage.Remove(documentOffset))
|
|
{
|
|
_documentCache.Remove(documentOffset);
|
|
_reverseCache.Remove(bsonDocument);
|
|
_indexTree.RemoveDocument(documentOffset);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
|
|
/* Document Cache */
|
|
|
|
public bool TryGetCachedDocument(long documentOffset, out BsonDocument cachedDocument) =>
|
|
_documentCache.TryGetValue(documentOffset, out cachedDocument);
|
|
|
|
public bool TryGetCachedDocumentOffset(BsonDocument bsonDocument, out long documentOffset) =>
|
|
_reverseCache.TryGetValue(bsonDocument, out documentOffset);
|
|
|
|
public bool TryAddOrUpdateCachedDocument(long documentOffset, BsonDocument bsonDocument)
|
|
{
|
|
if (_documentCache.TryGetValue(documentOffset, out BsonDocument cachedDocument))
|
|
{
|
|
_reverseCache.Remove(cachedDocument);
|
|
_documentCache.Remove(documentOffset);
|
|
}
|
|
_documentCache.Add(documentOffset, bsonDocument);
|
|
_reverseCache.Add(bsonDocument, documentOffset);
|
|
|
|
return true;
|
|
}
|
|
|
|
/* End Document Cache */
|
|
|
|
/* Index methods */
|
|
|
|
private void SetupIndeces()
|
|
{
|
|
bool rebuildNeeded = false;
|
|
|
|
foreach (var indexConfiguration in _configuration.Indeces)
|
|
{
|
|
if (!_indexTree.EnsureIndex(indexConfiguration.Key, indexConfiguration.Value, out IBsonIndex bsonIndex))
|
|
throw new ArgumentOutOfRangeException("duplicate index path");
|
|
|
|
string indexFileName =
|
|
String.Format("{0}.{1}.idx", _configuration.BaseFileName, indexConfiguration.Key);
|
|
|
|
if (rebuildNeeded || !File.Exists(indexFileName) || !LoadIndexFile(indexFileName, bsonIndex))
|
|
rebuildNeeded = true;
|
|
}
|
|
|
|
if (rebuildNeeded)
|
|
RebuildIndeces();
|
|
}
|
|
|
|
void RebuildIndeces()
|
|
{
|
|
_configuration.LogTarget("rebuilding indeces");
|
|
_indexTree.Clear();
|
|
|
|
foreach (BsonDocument bsonDocument in this)
|
|
{
|
|
if (!_reverseCache.TryGetValue(bsonDocument, out long documentOffset))
|
|
throw new ApplicationException("serious bug! no document offset");
|
|
_indexTree.AddDocument(documentOffset, bsonDocument);
|
|
}
|
|
}
|
|
|
|
private void UpdateIndex(BsonDocument bsonDocument)
|
|
{
|
|
if (TryGetCachedDocumentOffset(bsonDocument, out long documentOffset))
|
|
UpdateIndex(bsonDocument, documentOffset);
|
|
else
|
|
throw new KeyNotFoundException();
|
|
}
|
|
|
|
private void UpdateIndex(BsonDocument bsonDocument, long documentOffset)
|
|
{
|
|
_indexTree.AddDocument(documentOffset, bsonDocument);
|
|
}
|
|
|
|
public IEnumerable<BsonDocument> Query(String path, QueryOperator op, BsonValue value)
|
|
{
|
|
HashSet<long> resultsSet = new HashSet<long>();
|
|
_indexTree.Query(path, op, value.GetBytes(), resultsSet);
|
|
return new BsonDocumentEnumerable(() => new BsonDocumentEnumerator(this, resultsSet.GetEnumerator()));
|
|
}
|
|
|
|
private void SaveIndeces()
|
|
{
|
|
_configuration.LogTarget("saving indeces");
|
|
|
|
foreach (var indexConfiguration in _configuration.Indeces)
|
|
{
|
|
string indexFileName =
|
|
String.Format("{0}.{1}.idx", _configuration.BaseFileName, indexConfiguration.Key);
|
|
|
|
if (_indexTree.EnsureIndex(indexConfiguration.Key, indexConfiguration.Value, out IBsonIndex bsonIndex))
|
|
throw new ArgumentOutOfRangeException("unknown index path");
|
|
|
|
if (!SaveIndexFile(indexFileName, bsonIndex))
|
|
throw new Exception(String.Format("Failed to save index {0} to file {1}", indexConfiguration.Key, indexFileName));
|
|
}
|
|
}
|
|
|
|
bool LoadIndexFile(string indexFileName, IBsonIndex index)
|
|
{
|
|
try
|
|
{
|
|
using (FileStream indexFile = new FileStream(indexFileName, FileMode.Open, FileAccess.Read))
|
|
{
|
|
Guid indexGuid = new Guid(indexFile.ReadBytes(16));
|
|
if (!indexGuid.Equals(_fileStorage.CurrentGuid))
|
|
return false;
|
|
|
|
while (indexFile.Position < indexFile.Length)
|
|
{
|
|
long o = indexFile.ReadLong();
|
|
int l = indexFile.ReadInteger();
|
|
byte[] v = indexFile.ReadBytes(l);
|
|
index.AddValue(o, v);
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
catch (Exception)
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
bool SaveIndexFile(string indexFileName, IBsonIndex index)
|
|
{
|
|
using (FileStream indexFile = new FileStream(indexFileName, FileMode.Create, FileAccess.Write))
|
|
{
|
|
indexFile.WriteBytes(_fileStorage.CurrentGuid.ToByteArray());
|
|
|
|
foreach (var ie in index)
|
|
{
|
|
indexFile.WriteLong(ie.Key);
|
|
indexFile.WriteInteger(ie.Value.Length);
|
|
indexFile.WriteBytes(ie.Value);
|
|
}
|
|
indexFile.Flush();
|
|
indexFile.Close();
|
|
}
|
|
return true;
|
|
}
|
|
|
|
|
|
/* End Index methods */
|
|
|
|
public void Dispose()
|
|
{
|
|
Close();
|
|
|
|
_fileStorage?.Dispose();
|
|
_fileStorage = null;
|
|
}
|
|
|
|
public class BsonDocumentEnumerable : IEnumerable<BsonDocument>
|
|
{
|
|
private Func<IEnumerator<BsonDocument>> _getenumerator;
|
|
|
|
public BsonDocumentEnumerable(Func<IEnumerator<BsonDocument>> getenumerator)
|
|
{
|
|
_getenumerator = getenumerator;
|
|
}
|
|
|
|
public IEnumerator<BsonDocument> GetEnumerator() => _getenumerator();
|
|
IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
|
|
}
|
|
|
|
public IEnumerator<BsonDocument> GetEnumerator() => new BsonDocumentEnumerator(this);
|
|
IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
|
|
class BsonDocumentEnumerator : IEnumerator<BsonDocument>
|
|
{
|
|
private BsonDocumentStorage _documentStorage;
|
|
private IEnumerator<long> OffsetEnumerator;
|
|
|
|
public BsonDocumentEnumerator(BsonDocumentStorage documentStorage)
|
|
{
|
|
_documentStorage = documentStorage;
|
|
OffsetEnumerator = _documentStorage._fileStorage.Offsets.GetEnumerator();
|
|
}
|
|
|
|
public BsonDocumentEnumerator(BsonDocumentStorage documentStorage, IEnumerator<long> offsetEnumerator)
|
|
{
|
|
_documentStorage = documentStorage;
|
|
OffsetEnumerator = offsetEnumerator;
|
|
}
|
|
|
|
public bool MoveNext() => OffsetEnumerator.MoveNext();
|
|
public void Reset() => OffsetEnumerator.Reset();
|
|
|
|
public BsonDocument Current => _documentStorage.LoadDocument(OffsetEnumerator.Current);
|
|
object IEnumerator.Current => Current;
|
|
|
|
public void Dispose()
|
|
{
|
|
OffsetEnumerator?.Dispose();
|
|
OffsetEnumerator = null;
|
|
}
|
|
}
|
|
|
|
|
|
}
|
|
} |