LupusNobilis
/
ln.types
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

WIP

dev_timestamp
Harald Wolff 2019-03-29 08:55:17 +01:00
parent 536fe8d17b
commit aebc6df9ab
23 changed files with 1192 additions and 504 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

186
BTree.cs
View File

@ -1,186 +0,0 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
namespace ln.types
{
public class BTree<T> : IEnumerable<T>
{
public Comparison<T> Comparison { get; }
private TreeNode<T> headNode;
public BTree()
{
if (!typeof(T).GetInterfaces().Contains(typeof(IComparable<T>)))
throw new ArgumentException("BTree need to be constructed with Comparer<T> if T is not providing IComparable<T>");
Comparison = (T x, T y) => ((IComparable<T>)x).CompareTo(y);
}
public BTree(Comparison<T> comparison)
{
Comparison = comparison;
}
public void Add(T element)
{
}
public void Remove(T element)
{
}
public bool Contains(T element)
{
return false;
}
private TreeNode<T> First()
{
if (headNode == null)
return null;
return headNode.First();
}
private TreeNode<T> Last()
{
if (headNode == null)
return null;
return headNode.Last();
}
public IEnumerator<T> GetEnumerator()
{
TreeNode<T> node = First();
while (node != null)
{
yield return node.Value;
node = node.Next();
}
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
public class TreeNode<NT>
{
public BTree<NT> Tree { get; }
public NT Value { get; set; }
public TreeNode<NT> Parent { get; private set; }
public int Depth => 1 + (leftDepth > rightDepth ? leftDepth : rightDepth);
private TreeNode<NT> left;
private TreeNode<NT> right;
private int leftDepth;
private int rightDepth;
public TreeNode(BTree<NT> tree)
{
Tree = tree;
}
public TreeNode(BTree<NT> tree,NT value)
:this(tree)
{
Value = value;
}
private void Attach(TreeNode<NT> parent)
{
if (this.Parent == null)
Parent = parent;
else
throw new ArgumentException("TreeNode is already attached");
}
private void Detach(TreeNode<NT> parent)
{
if (this.Parent != parent)
Parent = null;
else
throw new ArgumentException("TreeNode is not attached to this parent");
}
public TreeNode<NT> First()
{
if (Left != null)
return Left.First();
return this;
}
public TreeNode<NT> Last()
{
if (Right != null)
return Left.Last();
return this;
}
public TreeNode<NT> Next()
{
if (Right != null)
return Right.First();
if (Parent == null)
return null;
TreeNode<NT> next = Parent;
while (Tree.Comparison(next.Value, this.Value) < 0)
next = next.Parent;
return next;
}
public TreeNode<NT> Left
{
get => left;
set
{
if (value == null)
{
if (left != null)
{
left.Detach(this);
leftDepth = 0;
}
}
else
{
if (left != null)
throw new ArgumentException("There is already a TreeNode attached to the left.");
value.Attach(this);
left = value;
leftDepth = left.Depth;
}
}
}
public TreeNode<NT> Right
{
get => right;
set
{
if (value == null)
{
if (right != null)
{
right.Detach(this);
rightDepth = 0;
}
}
else
{
if (right != null)
throw new ArgumentException("There is already a TreeNode attached to the left.");
value.Attach(this);
right = value;
rightDepth = right.Depth;
}
}
}
}
}
}

2
CIDR.cs
View File

@ -139,8 +139,8 @@ namespace ln.types
CIDR[] result = new CIDR[count];
for (int n = 0; n < count; n++)
{
nip += (uint)(1 << (32 - MaskWidth - bits));
result[n] = new CIDR(nip, newmask);
nip += (uint)(1 << (32 - MaskWidth - bits));
}
return result;
}

411
btree/BTree.cs 100644
View File

@ -0,0 +1,411 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using ln.types.serialize;
using System.Diagnostics.PerformanceData;
using System.Net.NetworkInformation;
namespace ln.types.btree
{
public class BTree<K,V>
{
public Comparison<K> Comparison { get; }
public int Count => count;
TreeNode headNode;
private int count;
public BTree()
{
if (!typeof(K).GetInterfaces().Contains(typeof(IComparable<K>)))
throw new ArgumentException("BTree need to be constructed with Comparer<K,V> if T is not providing IComparable<K,V>");
Comparison = (K x, K y) => ((IComparable<K>)x).CompareTo(y);
}
public BTree(Comparison<K> comparison)
{
Comparison = comparison;
}
public V this[K key]
{
get
{
TreeNode node = Find(key);
if (node == null)
throw new KeyNotFoundException();
return node.Value;
}
set
{
TreeNode node = Find(key);
if (node == null)
{
Add(key, value);
}
else
{
node.Value = value;
}
}
}
public void Add(K key,V value)
{
TreeNode treeNode = new TreeNode(this, key, value);
Insert(treeNode);
count++;
}
public void Remove(K key)
{
TreeNode node = Find(key);
if (node == null)
throw new KeyNotFoundException();
Remove(node);
count--;
}
public bool TryRemove(K key)
{
TreeNode node = Find(key);
if (node != null)
Remove(node);
return node != null;
}
public bool ContainsKey(K key)
{
return Find(key) != null;
}
public bool ContainsValue(V value)
{
return Values.Contains(value);
}
public void Clear()
{
headNode = null;
count = 0;
}
public K First()
{
TreeNode node = First(headNode);
if (node != null)
return node.Key;
throw new KeyNotFoundException("BTree is empty");
}
public K Last()
{
TreeNode node = Last(headNode);
if (node != null)
return node.Key;
throw new KeyNotFoundException("BTree is empty");
}
public K Previous(K current)
{
TreeNode node = Previous(Find(current));
if (node != null)
return node.Key;
throw new KeyNotFoundException();
}
public K Next(K current)
{
TreeNode node = Next(Find(current));
if (node != null)
return node.Key;
throw new KeyNotFoundException();
}
private TreeNode First(TreeNode node)
{
if (node == null)
return null;
while (node.Left != null)
node = node.Left;
return node;
}
private TreeNode Last(TreeNode node)
{
if (node == null)
return null;
while (node.Right != null)
node = node.Right;
return node;
}
private TreeNode Find(K key)
{
TreeNode node = headNode;
while (node != null)
{
int comp = Comparison(key, node.Key);
if (comp == 0)
return node;
else if (comp < 0)
node = node.Left;
else
node = node.Right;
}
return null;
}
private TreeNode Previous(TreeNode node)
{
if (node.Left != null)
return Last(node.Left);
while (node != null)
{
if (node.Parent == null)
return null;
if (node.Parent.Right == node)
return node.Parent;
node = node.Parent;
}
return null;
}
private TreeNode Next(TreeNode node)
{
if (node.Right != null)
return First(node.Right);
while (node != null)
{
if (node.Parent == null)
return null;
if (node.Parent.Left == node)
return node.Parent;
node = node.Parent;
}
return null;
}
private void Insert(TreeNode newNode)
{
if (headNode == null)
{
headNode = newNode;
}
else
{
TreeNode node = headNode;
while (true)
{
int c = Comparison(newNode.Key, node.Key);
if (c == 0)
{
throw new ArgumentException("Key exists");
}
else if (c < 0)
{
if (node.Left == null)
{
newNode.Attach(node);
Balance(node.Parent);
return;
}
else
node = node.Left;
}
else
{
if (node.Right == null)
{
newNode.Attach(node);
Balance(node.Parent);
return;
}
else
node = node.Right;
}
}
}
}
private void Remove(TreeNode node)
{
TreeNode p = node.Parent;
if ((node.Left != null) && (node.Right != null))
{
TreeNode replace = Last(node.Left);
node.Detach();
replace.Attach(p);
node.Left.Attach(replace);
node.Right.Attach(replace);
Balance(replace);
} else
{
node.Detach();
if (node.Left != null)
node.Left.Attach(p);
else if (node.Right != null)
node.Right.Attach(p);
Balance(p);
}
}
private void RotateRight(TreeNode node)
{
TreeNode l = node.Left;
TreeNode p = node.Parent;
node.Detach();
l.Attach(p);
if (l.Right!= null)
l.Right.Attach(node);
node.Attach(l);
}
private void RotateLeft(TreeNode node)
{
TreeNode r = node.Right;
TreeNode p = node.Parent;
node.Detach();
r.Attach(p);
if (r.Left != null)
r.Left.Attach(node);
node.Attach(r);
}
private void Balance(TreeNode node)
{
while (node != null)
{
TreeNode p = node.Parent;
int balance = node.LeftDepth - node.RightDepth;
if (balance < -1)
{
RotateLeft(node);
}
else if (balance > 1)
{
RotateRight(node);
}
node = p;
}
}
public IEnumerable<K> Keys
{
get
{
TreeNode node = First(headNode);
while (node != null)
{
yield return node.Key;
node = Next(node);
}
}
}
public IEnumerable<V> Values
{
get
{
TreeNode node = First(headNode);
while (node != null)
{
yield return node.Value;
node = Next(node);
}
}
}
class TreeNode
{
public BTree<K, V> Tree { get; }
public K Key { get; private set; }
public V Value { get; set; }
public TreeNode Parent { get; private set; }
public int Depth => 1 + Math.Max(LeftDepth, RightDepth);
public int LeftDepth => Left != null ? Left.Depth : 0;
public int RightDepth => Right != null ? Right.Depth : 0;
public TreeNode Left { get; private set; }
public TreeNode Right { get; private set; }
public TreeNode(BTree<K, V> tree, K key, V value)
{
Tree = tree;
Key = key;
Value = value;
}
public void Attach(TreeNode parent)
{
if (this.Parent != null)
Detach();
if (parent == null)
{
Tree.headNode = this;
Parent = null;
}
else
{
int comp = Tree.Comparison(Key, parent.Key);
if (comp == 0)
throw new ArgumentException(String.Format("Key already exists in BTree ({0})",Key));
else if (comp < 0)
{
if (parent.Left != null)
throw new InvalidOperationException("Another Key is already attached to Parent (left)");
parent.Left = this;
}
else
{
if (parent.Right != null)
throw new InvalidOperationException("Another Key is already attached to Parent (right)");
parent.Right = this;
}
Parent = parent;
}
}
public void Detach()
{
if (this.Parent != null)
{
if (this.Parent.Left == this)
this.Parent.Left = null;
else if (this.Parent.Right == this)
this.Parent.Right = null;
Parent = null;
}
}
public override string ToString()
{
return String.Format("[TreeNode Key={0} Value={1}]",Key,Value);
}
}
}
}

53
btree/MappingBTree.cs 100644
View File

@ -0,0 +1,53 @@
// /**
// * File: MappingBTree.cs
// * Author: haraldwolff
// *
// * This file and it's content is copyrighted by the Author and / or copyright holder.
// * Any use wihtout proper permission is illegal and may lead to legal actions.
// *
// *
// **/
using System;
using System.Runtime.CompilerServices;
using System.Collections.Generic;
using System.Collections;
namespace ln.types.btree
{
public delegate K KeyMapping<K, V>(V value);
public class MappingBTree<K,V> : IEnumerable<V>
{
public KeyMapping<K,V> KeyMapping { get; }
private BTree<K, V> tree = new BTree<K, V>();
public MappingBTree(KeyMapping<K,V> keyMapping)
{
KeyMapping = keyMapping;
}
public V this[K key]
{
get => tree[key];
set => tree[key] = value;
}
public int Count => tree.Count;
public void Add(V v) => tree.Add(KeyMapping(v), v);
public void Remove(V v) => tree.Remove(KeyMapping(v));
public bool TryRemove(V v) => tree.TryRemove(KeyMapping(v));
public bool Contains(V v) => tree.ContainsKey(KeyMapping(v));
public bool ContainsKey(K k) => tree.ContainsKey(k);
public IEnumerable<K> Keys => tree.Keys;
public IEnumerable<V> Values => tree.Values;
public V Next(V value) => tree[tree.Next(KeyMapping(value))];
public V Previous(V value) => tree[tree.Previous(KeyMapping(value))];
public IEnumerator<V> GetEnumerator() => Values.GetEnumerator();
IEnumerator IEnumerable.GetEnumerator() => Values.GetEnumerator();
}
}

11
ln.types.csproj
View File

@ -41,6 +41,9 @@
<Reference Include="LiteDB">
<HintPath>..\packages\LiteDB.4.1.4\lib\net40\LiteDB.dll</HintPath>
</Reference>
<Reference Include="nunit.framework">
<Package>nunit</Package>
</Reference>
</ItemGroup>
<ItemGroup>
<Compile Include="Properties\AssemblyInfo.cs" />
@ -83,7 +86,11 @@
<Compile Include="odb\values\ODBBool.cs" />
<Compile Include="odb\ODBCollection.DocumentIndex.DocumentIndexEntry.cs" />
<Compile Include="odb\ODBCollection.DocumentIndex.cs" />
<Compile Include="BTree.cs" />
<Compile Include="btree\BTree.cs" />
<Compile Include="btree\MappingBTree.cs" />
<Compile Include="odb\PropertyIndex.cs" />
<Compile Include="threads\Timing.cs" />
<Compile Include="test\testODB.cs" />
</ItemGroup>
<ItemGroup>
<Folder Include="sync\" />
@ -91,6 +98,8 @@
<Folder Include="odb\" />
<Folder Include="threads\" />
<Folder Include="odb\values\" />
<Folder Include="btree\" />
<Folder Include="test\" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\ln.logging\ln.logging.csproj">

16
odb/ODB.cs
View File

@ -23,7 +23,7 @@ namespace ln.types.odb
{
public String BasePath { get; set; }
Dictionary<string, ODBCollection> collections = new Dictionary<string, ODBCollection>();
internal Dictionary<string, ODBCollection> collections = new Dictionary<string, ODBCollection>();
public ODB(string basePath)
{
@ -35,7 +35,9 @@ namespace ln.types.odb
public ODBCollection GetCollection(string colName)
{
if (!collections.ContainsKey(colName))
collections[colName] = new ODBCollection(this, colName);
{
new ODBCollection(this, colName);
}
return collections[colName];
}
@ -48,9 +50,12 @@ namespace ln.types.odb
internal void DisposeCollection(ODBCollection collection)
{
ODBCollection check = collections[collection.CollectionName];
if (check == collection)
collections.Remove(collection.CollectionName);
if (collections.ContainsKey(collection.CollectionName))
{
ODBCollection check = collections[collection.CollectionName];
if (check == collection)
collections.Remove(collection.CollectionName);
}
}
private void Initialize()
@ -61,6 +66,7 @@ namespace ln.types.odb
{
foreach (ODBCollection col in collections.Values.ToArray())
{
col.Close();
col.Dispose();
}
}

236
odb/ODBCollection.DocumentIndex.DocumentIndexEntry.cs
View File

@ -9,242 +9,30 @@ namespace ln.types.odb
public partial class DocumentIndex
{
public class DocumentIndexEntry
public class Area
{
public DocumentIndex Index { get; }
public DocumentIndexEntry Next { get; private set; }
public DocumentIndexEntry Last { get; private set; }
public long Offset { get; private set; }
public int BufferLength { get; private set; }
public int Size { get; private set; }
public long NextOffset => Offset + BufferLength + 4;
public long NextOffset => Offset + Size + 4;
private ODBValue documentID;
public ODBValue DocumentID
public ODBValue DocumentID { get; set; }
public Area(long offset,int size)
{
get => documentID;
set
{
lock (Index)
{
if (!ODBNull.Instance.Equals(documentID))
{
Index.idLookup.Remove(documentID);
}
documentID = value;
if (!ODBNull.Instance.Equals(documentID))
{
if (Index.idLookup.ContainsKey(documentID))
{
throw new ArgumentException(String.Format("DocumentID {0} alread present",documentID));
}
Index.idLookup.Add(documentID, this);
}
}
}
}
public bool IsUnused => (ODBNull.Instance.Equals(documentID));
public DocumentIndexEntry(DocumentIndex index)
: this(index, 0)
{ }
private DocumentIndexEntry(DocumentIndex index, long offset)
{
Index = index;
Offset = offset;
Read();
Index.Log("NEWOFFSET {0}",this);
Size = size;
DocumentID = ODBNull.Instance;
}
private DocumentIndexEntry(DocumentIndexEntry lastEntry, int bufferLength)
public Area(long offset, int size,ODBValue documentID)
:this(offset,size)
{
Last = lastEntry;
Last.Next = this;
Index = lastEntry.Index;
Offset = lastEntry.NextOffset;
BufferLength = bufferLength;
WriteHeader();
Release();
Index.Log("FOLLOW {0}", Last);
Index.Log(" ==> {0}", this);
}
private DocumentIndexEntry(DocumentIndexEntry lastEntry,bool split = false)
{
Last = lastEntry;
Index = lastEntry.Index;
Offset = lastEntry.NextOffset;
if (split)
{
Next = Last.Next;
Last.Next = this;
if (Next != null)
{
Next.Last = this;
BufferLength = (int)(Next.Offset - Offset - 4);
}
else
{
BufferLength = (int)(Index.StorageStream.Length - Offset - 4);
}
WriteHeader();
Release();
Index.Log("INSERT {0}", Last);
Index.Log(" ==> {0}", this);
}
else
{
Last.Next = this;
Read();
Index.Log("NEXT {0}", this);
}
}
private void Read()
{
lock (Index)
{
if (Offset >= Index.StorageStream.Length)
{
BufferLength = 0;
DocumentID = ODBNull.Instance;
}
else
{
Index.StorageStream.Position = Offset;
BufferLength = Index.StorageStream.ReadInteger();
Index.StorageStream.Position = Offset + 4;
DocumentID = ODBValue.Read(Index.StorageStream);
if (NextOffset < Index.StorageStream.Length)
{
Next = new DocumentIndexEntry(this);
}
}
}
}
public void Update(ODBValue id, byte[] storageBytes)
{
lock (Index)
{
if (BufferLength < storageBytes.Length)
throw new ArgumentOutOfRangeException();
if (BufferLength > (storageBytes.Length + 32))
{
Split(storageBytes.Length);
}
Index.StorageStream.Position = Offset + 4;
Index.StorageStream.Write(storageBytes, 0, storageBytes.Length);
DocumentID = id;
Index.Log("UPDATE {0}", this);
}
}
public void Release()
{
lock (Index)
{
DocumentID = ODBNull.Instance;
Index.StorageStream.Position = Offset + 4;
Index.StorageStream.Write(new byte[BufferLength], 0, BufferLength);
Index.Log("RELEASE {0}", this);
if ((Next != null) && Next.IsUnused)
{
Combine();
}
if ((Last != null) && Last.IsUnused)
{
Last.Combine();
}
}
}
public DocumentIndexEntry Extend(int length)
{
if (Next == null)
{
if ((Last == null) && (IsUnused))
{
BufferLength = length;
WriteHeader();
Release();
Index.Log("IEXTEND {0}", this);
return this;
}
return new DocumentIndexEntry(this, length);
}
throw new NotSupportedException();
}
private void Split(int length)
{
Index.Log("SPLIT <== {0}", this);
BufferLength = length;
Index.Log(" ==> {0}", this);
DocumentIndexEntry dieInsert = new DocumentIndexEntry(this,true);
WriteHeader();
}
private void Combine()
{
Index.Log("COMBINE <== {0}", this);
Index.Log("COMBINE <== {0}", Next);
BufferLength += Next.BufferLength + 4;
Index.Log("COMBINE ==> {0}", this);
Next = Next.Next;
if (Next != null)
Next.Last = this;
WriteHeader();
}
private void WriteHeader()
{
lock (Index)
{
Index.StorageStream.Position = Offset;
Index.StorageStream.WriteInteger(BufferLength);
}
}
public byte[] ReadStorageBytes()
{
byte[] buffer = new byte[BufferLength];
lock (Index)
{
Index.StorageStream.Position = Offset + 4;
Index.StorageStream.Read(buffer, 0, BufferLength);
}
return buffer;
DocumentID = documentID;
}
public override string ToString()
{
return String.Format("[IndexEntry Offset=0x{0:x8} BufferLength=0x{1:x8} NextOffset=0x{2:X8}]",Offset,BufferLength,NextOffset);
return String.Format("[IndexEntry Offset=0x{0:x8} BufferLength=0x{1:x8} NextOffset=0x{2:X8}]",Offset,Size,NextOffset);
}
}

171
odb/ODBCollection.DocumentIndex.cs
View File

@ -3,6 +3,8 @@ using System.Collections.Generic;
using ln.types.odb.values;
using System.IO;
using ln.logging;
using ln.types.btree;
using System.Linq;
namespace ln.types.odb
{
@ -12,9 +14,11 @@ namespace ln.types.odb
{
public ODBCollection Collection { get; }
public Stream StorageStream { get; }
public DocumentIndexEntry Head { get; private set; }
private Dictionary<ODBValue, DocumentIndexEntry> idLookup = new Dictionary<ODBValue, DocumentIndexEntry>();
private Dictionary<ODBValue, Area> idLookup = new Dictionary<ODBValue, Area>();
private MappingBTree<long, Area> unusedAreas = new MappingBTree<long, Area>((d) => d.Offset);
private MappingBTree<long, Area> usedAreas = new MappingBTree<long, Area>((d) => d.Offset);
private FileLogger transactionLogger = null;
@ -23,9 +27,57 @@ namespace ln.types.odb
Collection = collection;
StorageStream = stream;
EnableTranslactionLog();
//EnableTransactionLog();
Head = new DocumentIndexEntry(this);
Initialize();
}
private void Initialize()
{
long offset = 0;
while (offset < StorageStream.Length)
{
StorageStream.Position = offset;
int asize = StorageStream.ReadInteger();
ODBValue documentID = ODBValue.Read(StorageStream);
Area area = new Area(offset, asize, documentID);
Log("INIT: O:0x{0:X8} S:0x{1:X8} N:0x{2:X8} {3}", area.Offset, area.Size, area.NextOffset, area.DocumentID);
if (ODBNull.Instance.Equals(documentID))
unusedAreas.Add(area);
else
{
if (idLookup.ContainsKey(documentID))
{
byte[] aBytes = Load(idLookup[documentID]);
ODBDocument aDoc = new ODBDocument(aBytes, 0, aBytes.Length);
byte[] bBytes = Load(area);
ODBDocument bDoc = new ODBDocument(bBytes, 0, bBytes.Length);
if (aDoc.StorageTimeStamp < bDoc.StorageTimeStamp)
{
WriteArea(idLookup[documentID], ODBNull.Instance, new byte[0]);
usedAreas.Add(area);
idLookup.Add(documentID, area);
}
else
{
WriteArea(area, ODBNull.Instance, new byte[0]);
}
}
else
{
usedAreas.Add(area);
idLookup.Add(documentID, area);
}
}
offset = area.NextOffset;
}
}
public void Close()
@ -37,7 +89,7 @@ namespace ln.types.odb
}
}
public void EnableTranslactionLog()
public void EnableTransactionLog()
{
if (transactionLogger == null)
{
@ -57,39 +109,106 @@ namespace ln.types.odb
transactionLogger.Message(LogLevel.INFO, transaction);
}
public IEnumerable<DocumentIndexEntry> IndexEntries
public bool Contains(ODBValue documentID)
{
get
return idLookup.ContainsKey(documentID);
}
public byte[] Load(ODBValue documentID)
{
lock (this)
{
DocumentIndexEntry entry = Head;
while (entry != null)
if (idLookup.ContainsKey(documentID))
{
yield return entry;
entry = entry.Next;
Area area = idLookup[documentID];
return Load(area);
}
return null;
}
}
public byte[] Load(Area area)
{
lock (this)
{
byte[] buffer = new byte[area.Size];
StorageStream.Position = area.Offset + 4;
StorageStream.Read(buffer, 0, area.Size);
return buffer;
}
}
public DocumentIndexEntry Lookup(ODBValue ID)
public void Store(ODBValue documentID, byte[] storageBytes)
{
foreach (DocumentIndexEntry die in IndexEntries)
lock (this)
{
if (ID.Equals(die.DocumentID))
return die;
Area previousArea = null;
Area storageArea = FindUnused(storageBytes.Length);
if (idLookup.ContainsKey(documentID))
previousArea = idLookup[documentID];
if (storageArea == null)
storageArea = new Area(StorageStream.Length, storageBytes.Length);
WriteArea(storageArea, documentID, storageBytes);
if (previousArea != null)
WriteArea(previousArea, ODBNull.Instance, new byte[0]);
}
}
public void Remove(ODBValue documentID)
{
lock (this)
{
Area area = idLookup[documentID];
WriteArea(area, ODBNull.Instance, new byte[0]);
}
}
private void WriteArea(Area area,ODBValue documentID,byte[] storageBytes)
{
Log("OFFSET(0x{0:X8}) SIZE(0x{1:X8}) ID: {2}",area.Offset,area.Size,documentID);
StorageStream.Position = area.Offset;
StorageStream.WriteInteger(area.Size);
StorageStream.Write(storageBytes, 0, storageBytes.Length);
if (storageBytes.Length < area.Size)
{
byte[] zero = new byte[area.Size - storageBytes.Length];
StorageStream.Write(zero, 0, zero.Length);
}
if ((ODBNull.Instance.Equals(documentID)) && (!ODBNull.Instance.Equals(area.DocumentID)))
{
usedAreas.Remove(area);
unusedAreas.Add(area);
if (idLookup[area.DocumentID] == area)
idLookup.Remove(area.DocumentID);
}
else if ((!ODBNull.Instance.Equals(documentID)) && (ODBNull.Instance.Equals(area.DocumentID)))
{
unusedAreas.TryRemove(area);
usedAreas.Add(area);
idLookup[documentID] = area;
}
area.DocumentID = documentID;
}
public IEnumerable<Area> UsedAreas => usedAreas.Values;
public Area Lookup(ODBValue documentID)
{
if (idLookup.ContainsKey(documentID))
return idLookup[documentID];
return null;
}
public DocumentIndexEntry FindUnused(int minLength)
public Area FindUnused(int minLength)
{
foreach (DocumentIndexEntry die in IndexEntries)
foreach (Area area in unusedAreas)
{
if (die.IsUnused && (die.BufferLength > minLength))
return die;
if (die.Next == null)
return die.Extend(minLength);
if (area.Size >= minLength)
return area;
}
return null;
}
@ -98,11 +217,7 @@ namespace ln.types.odb
{
return idLookup.Keys.GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
}
}

169
odb/ODBCollection.cs
View File

@ -21,9 +21,13 @@ namespace ln.types.odb
FileStream fileStream;
DocumentIndex documentIndex;
Dictionary<string, PropertyIndex> propertyIndeces = new Dictionary<string, PropertyIndex>();
internal ODBCollection(ODB odb,string collectionName)
{
ODB = odb;
odb.collections.Add(collectionName, this);
CollectionName = collectionName;
Initialize();
@ -31,7 +35,7 @@ namespace ln.types.odb
private void Initialize()
{
fileStream = new FileStream(Path.Combine(ODB.BasePath, String.Format("{0}.col",CollectionName)),FileMode.OpenOrCreate,FileAccess.ReadWrite);
fileStream = new FileStream(Path.Combine(ODB.BasePath, String.Format("{0}.col",CollectionName)),FileMode.OpenOrCreate,FileAccess.ReadWrite);
documentIndex = new DocumentIndex(fileStream,this);
}
@ -43,91 +47,144 @@ namespace ln.types.odb
public bool Insert(ODBDocument document)
{
DocumentIndex.DocumentIndexEntry die = documentIndex.Lookup(document.ID);
if (die == null)
if (Index.Contains(document.ID))
return false;
byte[] docBytes = document.ToStorage();
Index.Store(document.ID, docBytes);
foreach (PropertyIndex propIndex in propertyIndeces.Values)
{
byte[] docBytes = document.ToStorage();
die = documentIndex.FindUnused(docBytes.Length);
die.Update(document.ID,docBytes);
return true;
propIndex.Add(document[propIndex.Name], document.ID);
}
return false;
return true;
}
public bool Update(ODBDocument document)
{
DocumentIndex.DocumentIndexEntry die = documentIndex.Lookup(document.ID);
if (die != null)
{
byte[] docBytes = document.ToStorage();
if (die.BufferLength < docBytes.Length)
{
DocumentIndex.DocumentIndexEntry ndie = documentIndex.FindUnused(docBytes.Length);
die.DocumentID = ODBNull.Instance;
ndie.Update(document.ID, docBytes);
die.Release();
}
else
{
die.Update(document.ID, docBytes);
}
return true;
}
return false;
if (!Index.Contains(document.ID))
return false;
byte[] docBytes = document.ToStorage();
Index.Store(document.ID, docBytes);
return true;
}
public bool Upsert(ODBDocument document)
{
byte[] docBytes = document.ToStorage();
DocumentIndex.DocumentIndexEntry die = documentIndex.Lookup(document.ID);
DocumentIndex.DocumentIndexEntry rdie = null;
if ((die == null) || (die.BufferLength < docBytes.Length))
{
rdie = die;
die = documentIndex.FindUnused(docBytes.Length);
}
if (rdie != null)
rdie.DocumentID = ODBNull.Instance;
die.Update(document.ID, docBytes);
if (rdie != null)
rdie.Release();
Index.Store(document.ID, docBytes);
return true;
}
public ODBDocument GetDocumentByID(ODBValue id)
public void Delete(ODBValue documentID)
{
Logging.Log(LogLevel.DEBUG, "ODBCollection.GetDocumentByID(): {0}",id);
DocumentIndex.DocumentIndexEntry die = documentIndex.Lookup(id);
if (die != null)
ODBDocument document = GetDocumentByID(documentID);
foreach (PropertyIndex propIndex in propertyIndeces.Values)
{
byte[] storageBytes = die.ReadStorageBytes();
ODBDocument document = new ODBDocument(storageBytes, 0, storageBytes.Length);
return document;
propIndex.Remove(document[propIndex.Name], document.ID);
}
return null;
Index.Remove(documentID);
}
public ODBDocument GetDocumentByID(ODBValue documentID)
{
//Logging.Log(LogLevel.DEBUGDETAIL, "ODBCollection.GetDocumentByID(): {0}",documentID);
byte[] storageBytes = Index.Load(documentID);
ODBDocument document = new ODBDocument(storageBytes, 0, storageBytes.Length);
return document;
}
public IEnumerable<ODBValue> FindIDs(string propertyName, ODBValue value)
{
if (propertyIndeces.ContainsKey(propertyName))
{
foreach (ODBValue documentID in propertyIndeces[propertyName].Find(value))
{
if (documentID != null)
yield return documentID;
}
}
else
{
foreach (ODBDocument document in this)
{
if (value.Equals(document[propertyName]))
yield return document.ID;
}
}
}
public IEnumerable<ODBDocument> Find(string propertyName, ODBValue value)
{
foreach (ODBDocument document in this)
lock (this)
{
if (value.Equals(document[propertyName]))
yield return document;
if (propertyIndeces.ContainsKey(propertyName))
{
foreach (ODBValue documentID in propertyIndeces[propertyName].Find(value))
{
if (documentID != null)
yield return GetDocumentByID(documentID);
}
}
else
{
foreach (ODBDocument document in this)
{
if (value.Equals(document[propertyName]))
yield return document;
}
}
}
}
public ODBDocument FindOne(string propertyName, ODBValue value)
{
foreach (ODBDocument document in this)
lock (this)
{
if (value.Equals(document[propertyName]))
return document;
if (propertyIndeces.ContainsKey(propertyName))
{
foreach (ODBValue documentID in propertyIndeces[propertyName].Find(value))
{
if (documentID != null)
return GetDocumentByID(documentID);
}
}
else
{
foreach (ODBDocument document in this)
{
if (value.Equals(document[propertyName]))
return document;
}
}
}
return null;
}
public void EnsureIndex(string propertyName,string indexName = null)
{
if (indexName == null)
indexName = propertyName;
lock (this)
{
if (!propertyIndeces.ContainsKey(indexName))
{
PropertyIndex propertyIndex = new PropertyIndex(indexName);
propertyIndeces.Add(indexName, propertyIndex);
foreach (ODBDocument doc in this)
{
propertyIndex.Add(doc[propertyName], doc.ID);
}
}
}
}
public IEnumerator<ODBDocument> GetEnumerator()
{
foreach (ODBValue id in documentIndex.ToArray())

49
odb/ODBCollection<>.cs
View File

@ -3,6 +3,9 @@ using System.Collections;
using System.Collections.Generic;
using ln.types.odb.values;
using System.Linq;
using System.Reflection;
using System.Diagnostics;
using ln.types.threads;
namespace ln.types.odb
{
public class ODBCollection<T> : IEnumerable<T> where T:class
@ -20,6 +23,11 @@ namespace ln.types.odb
collection = new ODBCollection(odb, ElementType.FullName);
}
public void Close()
{
collection.Close();
}
public T this[ODBValue documentID]
{
get => Select(documentID);
@ -60,16 +68,25 @@ namespace ln.types.odb
}
public IEnumerable<T> Select(string fieldName, object value)
{
foreach (ODBDocument document in collection.Find(fieldName, ODBMapper.Default.MapValue(value)))
IEnumerable<ODBValue> documentIDs = collection.FindIDs(fieldName, ODBMapper.Default.MapValue(value)).ToArray();
foreach (ODBValue documentID in documentIDs)
{
yield return Select(document.ID);
yield return Select(documentID);
}
}
public IEnumerable<T> Select(Predicate<T> predicate)
{
return this.Where((x) => predicate(x));
return this.Where((x) => predicate(x)).ToArray();
}
public IEnumerable<T> Select(IEnumerable<ODBValue> documentIDs)
{
lock (this)
{
return documentIDs.Select(id => Select(id)).ToArray();
}
}
public T Select(ODBValue documentID)
{
lock (this)
@ -111,6 +128,32 @@ namespace ln.types.odb
}
}
public void Delete(T o)
{
lock (this)
{
ODBDocument document = ODBMapper.Default.MapValue(o) as ODBDocument;
collection.Delete(document.ID);
}
}
public void EnsureIndex(string name)
{
String backingFieldName = name;
PropertyInfo propertyInfo = ElementType.GetProperty(name, BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance);
if (propertyInfo != null)
{
backingFieldName = String.Format("<{0}>k__BackingField", name);
FieldInfo fieldInfo = ElementType.GetField(backingFieldName, BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance);
if (fieldInfo == null)
backingFieldName = name;
}
collection.EnsureIndex(backingFieldName,name);
}
public IEnumerator<T> GetEnumerator()
{
ODBValue[] documentIDs = null;

12
odb/ODBDocument.cs
View File

@ -15,6 +15,7 @@ namespace ln.types.odb
}
public ODBValue ID { get; set; } = new ODBGuid();
public DateTime StorageTimeStamp { get; private set; }
public ODBDocument(byte[] bytes,int offset,int length)
:this()
@ -22,6 +23,8 @@ namespace ln.types.odb
int endOffset = offset + length;
ID = ODBValue.Deserialize(bytes, ref offset);
StorageTimeStamp = ODBValue.Deserialize(bytes, ref offset).AsDateTime;
int nProps = BitConverter.ToInt32(bytes, offset);
offset += 4;
@ -59,6 +62,8 @@ namespace ln.types.odb
public IEnumerable<ODBValue> Keys => properties.Keys;
public override int CompareLevel => 128;
public bool Contains(ODBStringValue propName)
{
return !ODBNull.Instance.Equals(this[propName]);
@ -70,6 +75,8 @@ namespace ln.types.odb
BinaryWriter writer = new BinaryWriter(stream);
ID.Store(writer);
((ODBValue)DateTime.Now).Store(writer);
writer.Write(properties.Count);
foreach (ODBValue propName in properties.Keys)
@ -101,6 +108,11 @@ namespace ln.types.odb
return false;
}
public override int CompareInType(ODBValue other)
{
return ID.CompareTo(other.AsDocument.ID);
}
static ODBDocument()
{
RegisterDeserializer(0x1000, (b,o,l) => new ODBDocument(b,o,l));

63
odb/PropertyIndex.cs 100644
View File

@ -0,0 +1,63 @@
// /**
// * File: FieldIndex.cs
// * Author: haraldwolff
// *
// * This file and it's content is copyrighted by the Author and / or copyright holder.
// * Any use wihtout proper permission is illegal and may lead to legal actions.
// *
// *
// **/
using System;
using ln.types.btree;
using ln.types.odb.values;
using System.Collections;
using System.Collections.Generic;
namespace ln.types.odb
{
public class PropertyIndex
{
public String Name { get; }
BTree<ODBValue, List<ODBValue>> bTree;
public PropertyIndex(String name)
{
Name = name;
bTree = new BTree<ODBValue, List<ODBValue>>(ODBValue.Compare);
}
public IEnumerable<ODBValue> Find(ODBValue lookup)
{
if (bTree.ContainsKey(lookup))
{
foreach (ODBValue documentID in bTree[lookup])
yield return documentID;
}
}
public void Add(ODBValue value,ODBValue documentID)
{
List<ODBValue> indList = null;
if (!bTree.ContainsKey(value))
{
indList = new List<ODBValue>();
bTree[value] = indList;
}
else
{
indList = bTree[value];
}
indList.Add(documentID);
}
public void Remove(ODBValue value,ODBValue documentID)
{
if (bTree.ContainsKey(value))
{
bTree[value].Remove(documentID);
}
}
}
}

10
odb/values/ODBBool.cs
View File

@ -12,6 +12,8 @@ namespace ln.types.odb.values
{
public class ODBBool : ODBValue
{
public override int CompareLevel => 254;
public ODBBool()
:base(0x04)
{}
@ -27,6 +29,14 @@ namespace ln.types.odb.values
return new byte[] { AsBool ? (byte)0xFF : (byte)0x00 };
}
public override int CompareInType(ODBValue other)
{
if (AsBool == other.AsBool)
return 0;
if (AsBool)
return 1;
return -1;
}
static ODBBool()
{

14
odb/values/ODBDouble.cs
View File

@ -3,6 +3,8 @@ namespace ln.types.odb.values
{
public class ODBDouble : ODBValue
{
public override int CompareLevel => 2;
public ODBDouble()
:base(0x18)
{
@ -18,6 +20,18 @@ namespace ln.types.odb.values
return BitConverter.GetBytes(AsDouble);
}
public override int CompareInType(ODBValue other)
{
double a = AsDouble;
double b = other.AsDouble;
if (Math.Abs(a - b) < double.Epsilon)
return 0;
if (a < b)
return -1;
return 1;
}
static ODBDouble()
{
RegisterDeserializer(0x0018, (b, o, l) => BitConverter.ToDouble(b, o));

7
odb/values/ODBGuid.cs
View File

@ -4,6 +4,8 @@ namespace ln.types.odb.values
{
public class ODBGuid : ODBValue
{
public override int CompareLevel => 20;
public ODBGuid()
:base(0x03)
{
@ -27,6 +29,11 @@ namespace ln.types.odb.values
return new Guid(s);
}
public override int CompareInType(ODBValue other)
{
return AsGuid.CompareTo(other.AsGuid);
}
static ODBGuid()
{
RegisterDeserializer(0x03, (b,o,l) => FromByteArray(b,o));

19
odb/values/ODBInteger.cs
View File

@ -16,8 +16,15 @@ namespace ln.types.odb.values
public override bool AsBool => AsInt != 0;
public override int CompareLevel => 0;
public override byte[] ToStorage() => BitConverter.GetBytes(AsInt);
public override int CompareInType(ODBValue other)
{
return AsInt - other.AsInt;
}
static ODBInteger()
{
RegisterDeserializer(0x10, (b, o, l) => BitConverter.ToInt32(b, o));
@ -29,6 +36,8 @@ namespace ln.types.odb.values
}
public class ODBUInteger : ODBValue
{
public override int CompareLevel => 5;
public ODBUInteger()
: base(0x11)
{
@ -41,6 +50,16 @@ namespace ln.types.odb.values
public override byte[] ToStorage() => BitConverter.GetBytes(AsUInt);
public override int CompareInType(ODBValue other)
{
long d = AsUInt - other.AsUInt;
if (d == 0)
return 0;
if (d < 0)
return -1;
return 1;
}
static ODBUInteger()
{
RegisterDeserializer(0x11, (b, o, l) => BitConverter.ToUInt32(b, o));

8
odb/values/ODBList.cs
View File

@ -56,6 +56,8 @@ namespace ln.types.odb.values
}
}
public override int CompareLevel => 253;
public override byte[] ToStorage()
{
MemoryStream stream = new MemoryStream();
@ -69,6 +71,12 @@ namespace ln.types.odb.values
return stream.ToArray();
}
public override int CompareInType(ODBValue other)
{
// ToDO: Implement List comparison
return 0;
}
static ODBList()
{
RegisterDeserializer(0x02, (b, o, l) => new ODBList(b,o,l));

29
odb/values/ODBLong.cs
View File

@ -16,9 +16,24 @@ namespace ln.types.odb.values
public override byte[] ToStorage() => BitConverter.GetBytes(AsLong);
public override int CompareInType(ODBValue other)
{
long a, b;
a = AsLong;
b = other.AsLong;
long d = a - b;
if (d == 0)
return 0;
if (d < 0)
return -1;
return 1;
}
public override DateTime AsDateTime => DateTimeOffset.FromUnixTimeMilliseconds(AsLong).DateTime;
public override TimeSpan AsTimeSpan => TimeSpan.FromMilliseconds(AsDouble);
public override int CompareLevel => 1;
static ODBLong()
{
@ -30,6 +45,8 @@ namespace ln.types.odb.values
}
public class ODBULong : ODBValue
{
public override int CompareLevel => 6;
public ODBULong()
: base(0x13)
{
@ -42,6 +59,18 @@ namespace ln.types.odb.values
public override byte[] ToStorage() => BitConverter.GetBytes(AsULong);
public override int CompareInType(ODBValue other)
{
ulong a = AsULong;
ulong b = other.AsULong;
if (a == b)
return 0;
if (a < b)
return -1;
return 1;
}
static ODBULong()
{
RegisterDeserializer(0x13, (b, o, l) => BitConverter.ToUInt64(b, o));

7
odb/values/ODBNull.cs
View File

@ -6,6 +6,8 @@ namespace ln.types.odb.values
{
public static readonly ODBNull Instance = new ODBNull();
public override int CompareLevel => 255;
public ODBNull()
: base(0x00)
{ }
@ -28,5 +30,10 @@ namespace ln.types.odb.values
{
return (obj == null) || (obj is ODBNull);
}
public override int CompareInType(ODBValue other)
{
return 0;
}
}
}

7
odb/values/ODBStringValue.cs
View File

@ -1,9 +1,12 @@
using System;
using System.Text;
using System.Globalization;
namespace ln.types.odb.values
{
public class ODBStringValue : ODBValue
{
public override int CompareLevel => 10;
public ODBStringValue()
: base(0x01)
{ }
@ -19,6 +22,10 @@ namespace ln.types.odb.values
return Encoding.UTF8.GetBytes(AsString);
}
public override int CompareInType(ODBValue other)
{
return AsString.CompareTo(other.AsString);
}
public static implicit operator ODBStringValue(String text)
{

16
odb/values/ODBValue.cs
View File

@ -48,6 +48,9 @@ namespace ln.types.odb.values
Value = value;
}
public abstract int CompareLevel { get; }
public abstract int CompareInType(ODBValue other);
public abstract byte[] ToStorage();
public object AsObject => Value;
@ -146,9 +149,22 @@ namespace ln.types.odb.values
throw new FormatException("wrong storage type code");
}
public int CompareTo(ODBValue other)
{
if (CompareLevel != other.CompareLevel)
return CompareLevel - other.CompareLevel;
return CompareInType(other);
}
public override string ToString()
{
return String.Format("[ODBValue Value={0}]", Value);
}
public static int Compare(ODBValue a,ODBValue b)
{
return a.CompareTo(b);
}
}
}

153
test/testODB.cs 100644
View File

@ -0,0 +1,153 @@
// /**
// * File: ODB.cs
// * Author: haraldwolff
// *
// * This file and it's content is copyrighted by the Author and / or copyright holder.
// * Any use wihtout proper permission is illegal and may lead to legal actions.
// *
// *
// **/
using NUnit.Framework;
using System;
using System.IO;
using ln.types.odb;
using Newtonsoft.Json;
using System.Diagnostics;
using ln.types.threads;
using System.Threading;
using ln.logging;
namespace ln.types.test
{
[TestFixture()]
public class testODB
{
[Test()]
public void TestCase()
{
String odbBase = Path.Combine(Path.GetTempPath(), "odb.test");
Console.WriteLine("Using path {0}", odbBase);
if (Directory.Exists(odbBase))
Directory.Delete(odbBase, true);
Guid docIdOne;
using (ODB odb = new ODB(odbBase))
{
ODBCollection<ODBTestClassA> colA = odb.GetCollection<ODBTestClassA>();
int[] numbers = new int[] { 0,53,2,34,37,35,23,1,456,7777,43,3535367 };
foreach (int n in numbers)
{
ODBTestClassA ca = new ODBTestClassA(n);
colA.Insert(ca);
}
ODBTestClassA cdouble = colA.SelectOne("Number", 1);
Assert.NotNull(cdouble);
Assert.AreEqual(cdouble.Number, 1);
docIdOne = cdouble.ID;
bool success = colA.Insert(cdouble);
Assert.IsFalse(success);
success = colA.Update(cdouble);
Assert.IsTrue(success);
success = colA.Upsert(cdouble);
Assert.IsTrue(success);
colA.Close();
}
using (ODB odb = new ODB(odbBase))
{
ODBCollection<ODBTestClassA> colA = odb.GetCollection<ODBTestClassA>();
Timing.Meassure("EnsureIndex", () => colA.EnsureIndex("Number"));
ODBTestClassA cSelect = colA.SelectOne("Number", 1);
Assert.AreEqual(docIdOne, cSelect.ID);
colA.Delete(cSelect);
cSelect = colA.SelectOne("Number", 1);
Assert.IsNull(cSelect);
cSelect = colA.SelectOne("Number", 3);
Assert.IsNull(cSelect);
Random random = new Random();
int choose = random.Next() % 10000;
int selection = 0;
int lc = choose;
Thread t = new Thread(() => { while (true) { Thread.Sleep(1000); Logging.Log(LogLevel.INFO," ==> {0}", lc - choose); lc = choose; }; });
t.Start();
for (int m = 0; m < 5; m++)
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
for (int n = 0; n < 10000; n++)
{
ODBTestClassA testMass = new ODBTestClassA(random.Next());
colA.Upsert(testMass);
if (choose == 0)
selection = testMass.Number;
choose--;
}
stopwatch.Stop();
Console.WriteLine("Inserted 10000 Items in {0}ms", stopwatch.ElapsedMilliseconds);
}
Stopwatch stopwatch2 = new Stopwatch();
stopwatch2.Start();
ODBTestClassA selA = colA.SelectOne("Number", selection);
stopwatch2.Stop();
Console.WriteLine("Looked up item in {0}ms", stopwatch2.ElapsedMilliseconds);
t.Interrupt();
}
}
}
public class ODBTestClassA
{
[DocumentID]
public readonly Guid ID = Guid.NewGuid();
public String PropertyString { get; set; }
public string FieldString;
public int Number;
private ODBTestClassA()
{ }
public ODBTestClassA(int n)
{
Number = n;
PropertyString = String.Format("Property:{0}", n);
FieldString = String.Format("Field:{0}", n);
}
}
}

47
threads/Timing.cs 100644
View File

@ -0,0 +1,47 @@
// /**
// * File: Timing.cs
// * Author: haraldwolff
// *
// * This file and it's content is copyrighted by the Author and / or copyright holder.
// * Any use wihtout proper permission is illegal and may lead to legal actions.
// *
// *
// **/
using System;
using System.Diagnostics;
using ln.logging;
namespace ln.types.threads
{
public static class Timing
{
public delegate void VoidDelegate();
public delegate T TypedDelegate<T>();
public static void Meassure(VoidDelegate f) => Meassure("", f);
public static void Meassure(String prefix,VoidDelegate f)
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
f();
stopwatch.Stop();
Logging.Log(LogLevel.DEBUG, "Timing({1}): {0}ms", stopwatch.ElapsedMilliseconds,prefix);
}
public static T Meassure<T>(TypedDelegate<T> f)
{
return Meassure("", f);
}
public static T Meassure<T>(String prefix,TypedDelegate<T> f)
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
T r = f();
stopwatch.Stop();
Logging.Log(LogLevel.DEBUG, "Timing({1}): {0}ms", stopwatch.ElapsedMilliseconds,prefix);
return r;
}
}
}