using ln.logging; using ln.objects.index; using ln.objects.serialization; using ln.objects.storage; using ln.type; using System.Collections.Generic; using System.Linq; using System.Runtime.Serialization; using ln.objects.serialization.binary; using ln.collections; using System; namespace ln.objects { public class ObjectStore : IDisposable { public static bool DEBUG; public static Guid TYPECACHE_GUID = new Guid("00000000-0000-0000-0001-000000000001"); public static Guid SPECIAL_GUID_LIMIT = new Guid("00000000-0000-0000-0002-000000000000"); public String FileName => objectFile.FileName; public Factory SerializerFactory { get; } public Factory DeserializerFactory { get; } Deserializer defaultDeserializer; BinaryObjectFile objectFile; ObjectCache objectCache; BTreeValueSet objectTypeCache; Dictionary indeces = new Dictionary(); SaveTransaction currentTransaction; public ObjectStore(String filename) :this(filename,() => new BinarySerializer(),() => new BinaryDeserializer()) { } public ObjectStore(string filename, Factory serializerFactory, Factory deserializerFactory) : this(filename, (o) => serializerFactory(), (o) => deserializerFactory()) { } public ObjectStore(string filename, Factory serializerFactory, Factory deserializerFactory) { SerializerFactory = serializerFactory; DeserializerFactory = deserializerFactory; objectFile = new BinaryObjectFile(filename); objectCache = new ObjectCache(); } public void Open() { lock (this) { objectFile.Open(); defaultDeserializer = DeserializerFactory(this); defaultDeserializer.OnLookupObjectByReference += LookupObjectByReference; if (objectFile.TryReadBinaryObject(TYPECACHE_GUID, out byte[] typeCacheBytes)) { object otc = null; if (defaultDeserializer.DeserializeObject(typeCacheBytes, ref otc)) { KeyValuePair[] typeCacheValues = (KeyValuePair[])otc; objectTypeCache = new BTreeValueSet(); objectTypeCache.AddRange(typeCacheValues.Select((kvp) => new KeyValuePair(Type.GetType(kvp.Key), kvp.Value))); objectFile.RemoveBinaryObjects(TYPECACHE_GUID); objectFile.Flush(); } else { objectFile.RemoveBinaryObjects(TYPECACHE_GUID); objectFile.Flush(); objectTypeCache = new BTreeValueSet(); foreach (Guid uid in objectFile.ObjectUIDs) { if (objectFile.TryReadBinaryObject(uid, out byte[] serializedBytes)) { if (defaultDeserializer.TryGetType(serializedBytes, out Type type)) { objectTypeCache.TryAdd(type, uid); } } } } } else { objectTypeCache = new BTreeValueSet(); } } } public void Close() { lock (this) { if (objectFile != null) { KeyValuePair[] typeCacheValues = objectTypeCache.GetKeyValuePairs().Select((kvp) => new KeyValuePair(kvp.Key.GetSimpleQualifiedName(), kvp.Value)).ToArray(); if (SerializerFactory(this).SerializeObject(typeCacheValues, out byte[] typeCacheBytes)) { objectFile.WriteBinaryObject(TYPECACHE_GUID, typeCacheBytes); objectFile.Flush(); } objectFile.Dispose(); objectFile = null; } } } public T LoadObject(Guid uid) => (T)LoadObject(uid, typeof(T)); public object LoadObject(Guid uid,Type type) { lock (this) { if (!objectCache.TryGetObject(uid,out object o)) { o = Activator.CreateInstance(type, true); objectCache.Add(uid, o); byte[] boData = objectFile.ReadBinaryObject(uid); if (!defaultDeserializer.DeserializeObject(boData, ref o)) throw new Exception("unable to deserialize"); } return o; } } public IEnumerable LoadObjects(Type type) { foreach (Guid uid in objectTypeCache[type]) { yield return LoadObject(uid, type); } } public IEnumerable LoadObjects() { foreach (Guid uid in objectTypeCache[typeof(T)]) { yield return LoadObject(uid); } } private bool LookupObjectByReference(object reference, Type targetType, out object o) { if (DEBUG) Logging.Log(LogLevel.DEBUG, "LookupObjectReference: {0} [ {1} ]", reference, targetType.Name); o = LoadObject((Guid)reference, targetType); return true; } private void TransactionalAction(SaveTransaction saveTransaction, Action action) { lock (this) { SaveTransaction oldTransaction = currentTransaction; currentTransaction = saveTransaction; try { action(); } finally { currentTransaction = oldTransaction; } } } public Guid SaveObject(object o) => SaveObject(GetObjectUID(o), o); public Guid SaveObject(Guid uid, object o) { lock (this) { SaveTransaction saveTransaction = new SaveTransaction(this); TransactionalAction(saveTransaction, () => PrepareSave(saveTransaction, uid, o)); foreach (KeyValuePair bo in saveTransaction.BinaryObjects) if (bo.Value != null) { if (DEBUG) Logging.Log(LogLevel.DEBUG, "Storing {0} [ {1} ]", bo.Key, objectCache[bo.Key].GetType()); objectFile.WriteBinaryObject(bo.Key, bo.Value); } else if (DEBUG) Logging.Log(LogLevel.DEBUG, "Ignoring unchanged {0} [ {1} ]", bo.Key, objectCache[bo.Key].GetType()); return uid; } } private void PrepareSave(SaveTransaction saveTransaction,Guid uid, object o) { saveTransaction.PushReferencedObjects(); if (!saveTransaction.Serializer.SerializeObject(o, out byte[] serializedBytes)) throw new SerializationException(); if (!Object.ReferenceEquals(null, o)) { objectTypeCache.TryAdd(o.GetType(), uid); GetIndexLeaf(o.GetType()).Reindex(uid, o); } if (objectFile.TryReadBinaryObject(uid, out byte[] storedBytes) && storedBytes.AreEqual(serializedBytes)) { saveTransaction.BinaryObjects.Add(uid, null); saveTransaction.PopReferencedObjects(); } else { saveTransaction.BinaryObjects.Add(uid, serializedBytes); saveTransaction.PopReferencedObjectsToFinal(); } foreach (object ro in saveTransaction.ReferencedObjects.ToArray()) { Guid oUid = GetObjectUID(ro); if (!saveTransaction.BinaryObjects.ContainsKey(oUid)) PrepareSave(saveTransaction, oUid, ro); } } public Guid GetObjectUID(object o) { lock (this) { if (!objectCache.TryGetUID(o, out Guid uid)) { uid = Guid.NewGuid(); objectCache.Add(uid, o); } return uid; } } private IndexLeaf GetIndexLeaf(Type type) { if (!indeces.TryGetValue(type,out IndexLeaf indexLeaf)) { indexLeaf = new IndexLeaf(type, null, (o) => o); indeces.Add(type, indexLeaf); } return indexLeaf; } public bool EnsureIndeces(Type type,IEnumerable> indexDefinitions) { bool rebuild = false; foreach (KeyValuePair indexDefinition in indexDefinitions) rebuild |= EnsureIndex(type, indexDefinition.Key, indexDefinition.Value, false); if (rebuild) RebuildIndeces(); return rebuild; } public IEnumerable QueryObjects(Type type, string path, Func criterion) => QueryUids(type, path, criterion).Select((uid) => LoadObject(uid, type)); public IEnumerable QueryUids(Type type,string path,Func criterion) { HashSet result = new HashSet(); GetIndexLeaf(type).GetLeaf(path).Match(criterion, result); return result; } public bool EnsureIndex(Type type, string path, index.Index index) => EnsureIndex(type, path, index, true); private bool EnsureIndex(Type type, string path, index.Index index, bool rebuild) { IndexLeaf indexLeaf = GetIndexLeaf(type).GetLeaf(path); if (index.GetType().Equals(indexLeaf.Index)) return false; indexLeaf.Index = index; if (rebuild) RebuildIndeces(); return true; } private void RebuildIndeces() { foreach (Type type in objectTypeCache.Keys) { IndexLeaf indexLeaf = GetIndexLeaf(type); indexLeaf.Clear(); foreach (object value in LoadObjects(type)) indexLeaf.Reindex(GetObjectUID(value), value); } } public void Dispose() { Close(); } class SaveTransaction { public ObjectStore ObjectStore { get; } public Serializer Serializer { get; } public HashSet ReferencedObjects { get; } = new HashSet(); public Stack> referencedObjectsStack { get; } = new Stack>(); HashSet currentReferencedObjectsSet => referencedObjectsStack.Peek(); public Dictionary BinaryObjects { get; } = new Dictionary(); public SaveTransaction(ObjectStore objectStore) { ObjectStore = objectStore; Serializer = ObjectStore.SerializerFactory(objectStore); Serializer.OnLookupReference += LookupReference; } public bool LookupReference(object value, out object reference) { reference = ObjectStore.GetObjectUID(value); currentReferencedObjectsSet.Add(value); return true; } public void PushReferencedObjects() => referencedObjectsStack.Push(new HashSet()); public void PopReferencedObjects() => referencedObjectsStack.Pop(); public void PopReferencedObjectsToFinal() { HashSet ro = referencedObjectsStack.Pop(); foreach (object o in ro) ReferencedObjects.Add(o); } } } }