// /** // * File: SegmentedFile.cs // * Author: haraldwolff // * // * This file and it's content is copyrighted by the Author and / or copyright holder. // * Any use wihtout proper permission is illegal and may lead to legal actions. // * // * // **/ using System; using System.Collections.Generic; using System.IO; using System.Linq; using ln.logging; using ln.types.btree; namespace ln.types.odb.ng.storage { /** * SegmentedFile * ---------- * 0000 4 MAGIC Bytes * 0004 4 Version * 0008 8 LastCloseTimestamp * 0010 4 FirstOffset * 0014 4 GranularWidth * 0018 8 Reserved 0 * **/ public class SegmentedFile { public static byte[] MagicBytes { get; } = new byte[] { 0x0F, 0x0E, 0x0D, 0x0A }; public String FileName { get; } public int FileVersion { get; private set; } public long LastCloseTimestamp { get; private set; } public int FirstOffset { get; private set; } public int GranularWidth { get; private set; } = 12; public int GranularityMask => (1 << GranularWidth) - 1; public int AppendOffset { get; private set; } public IEnumerable Segments => segments; MappingBTree segments = new MappingBTree((s) => s.Offset); FileStream fileStream; public SegmentedFile(string fileName) { FileName = fileName; } public SegmentedFile(string fileName,int granularWidth) :this(fileName) { GranularWidth = granularWidth; } private void AssertOpen() { if (fileStream == null) throw new IOException("FSStorage not opened"); } private void CheckGranularity(ref int i){ i = (i + GranularityMask) & ~GranularityMask; } public bool IsOpen => (fileStream != null); public bool Open() { if (!IsOpen) { try { fileStream = new FileStream(FileName, FileMode.OpenOrCreate); if (fileStream.Length == 0) { FileVersion = 0; LastCloseTimestamp = 0; FirstOffset = (1 << GranularWidth); if (FirstOffset < 0x20) throw new NotSupportedException("Granularity too small"); AppendOffset = FirstOffset; Close(); return Open(); } else { if (!fileStream.ReadBytes(4).SequenceEqual(MagicBytes)) throw new IOException("Magic bytes do not match"); FileVersion = fileStream.ReadInteger(); LastCloseTimestamp = fileStream.ReadLong(); FirstOffset = fileStream.ReadInteger(); GranularWidth = fileStream.ReadInteger(); Scan(); fileStream.Position = 8; fileStream.WriteLong(0); fileStream.Flush(); } } catch (Exception e) { Logging.Log(e); if (fileStream != null) { fileStream.Close(); fileStream.Dispose(); fileStream = null; } return false; } return true; } return false; } public Segment Append(Guid id,byte[] payload) => Append(id, payload.Length, payload); public Segment Append(Guid id, int dataSize) => Append(id, dataSize); public Segment Append(Guid id, int dataSize, byte[] payload) { dataSize += Segment.HeaderSize; CheckGranularity(ref dataSize); Segment segment = new Segment(AppendOffset, dataSize) { ID = id, }; Write(segment, payload); segments.Add(segment); AppendOffset = segment.NextOffset; return segment; } public Segment Join(Segment a,Segment b) { if (a.NextOffset != b.Offset) throw new ArgumentException("Segments to join are not siblings"); a.Size += b.Size; WriteSegmentHead(a); segments.Remove(b); return a; } public Segment Split(Segment segment,int dataSize) { int requestedSize = dataSize + Segment.HeaderSize; CheckGranularity(ref requestedSize); if (requestedSize < segment.Size) { Segment split = new Segment(segment.Offset + requestedSize,segment.Size - requestedSize); segment.Size = requestedSize; segments.Add(split); WriteSegmentHead(split); WriteSegmentHead(segment); return split; } return null; } public byte[] Read(Segment segment) { fileStream.Position = segment.PayloadOffset; return fileStream.ReadBytes(segment.PayloadSize); } private void WriteSegmentHead(Segment segment) { fileStream.Position = segment.Offset; fileStream.WriteInteger(segment.Size); fileStream.WriteBytes(segment.ID.ToByteArray()); fileStream.WriteDouble(segment.TimeStamp.ToUnixTimeMilliseconds()); } public void Write(Segment segment,byte[] bytes) { AssertOpen(); if (bytes.Length > (segment.PayloadSize)) throw new ArgumentOutOfRangeException(nameof(bytes)); segment.TimeStamp = DateTime.Now; WriteSegmentHead(segment); fileStream.Position = segment.PayloadOffset; fileStream.WriteBytes(bytes); fileStream.WriteBytes(new byte[segment.PayloadSize - bytes.Length]); } /** * Position fileStream to offset, read Segment Header and construct a Segment instance to return **/ private Segment ScanSegment(int offset) { fileStream.Position = offset; int size = fileStream.ReadInteger(); byte[] id = fileStream.ReadBytes(16); double timestamp = fileStream.ReadDouble(); return new Segment(offset, size, DateTimeExtensions.FromUnixTimeMilliseconds(timestamp)) { ID = new Guid(id), }; } /** * Start at First Segment Offset and scan for all Segments in file **/ private void Scan() { int offset = FirstOffset; Segment segment = null; while (offset < fileStream.Length) { segment = ScanSegment(offset); segments.Add(segment); offset = segment.NextOffset; } AppendOffset = offset; } public void Close() { lock (this){ AssertOpen(); fileStream.Position = 0; fileStream.WriteBytes(MagicBytes); fileStream.WriteInteger(FileVersion); LastCloseTimestamp = (long)DateTime.Now.ToUnixTimeMilliseconds(); fileStream.WriteLong(LastCloseTimestamp); fileStream.WriteInteger(FirstOffset); fileStream.WriteInteger(GranularWidth); fileStream.Close(); fileStream.Dispose(); fileStream = null; } } public void Sync() { lock (this) { fileStream.Flush(); } } public class Segment { public static readonly int HeaderSize = 32; public int Offset { get; } public int Size { get; set; } public int PayloadOffset => Offset + HeaderSize; public int PayloadSize => Size - HeaderSize; public Guid ID { get; set; } public DateTime TimeStamp { get; set; } public int NextOffset => Offset + Size; public Segment(int offset, int size) { Offset = offset; Size = size; } public Segment(int offset, int size,DateTime timestamp) :this(offset,size) { TimeStamp = timestamp; } public Segment Split(int splitSize) { if (splitSize >= Size) throw new ArgumentOutOfRangeException(nameof(splitSize)); Segment splitArea = new Segment(Offset + Size - splitSize, splitSize); Size -= splitSize; return splitArea; } public override string ToString() { return string.Format("[StorageArea Offset=0x{0:x8} Size=0x{1:x8}]", Offset, Size); } } } }