LupusNobilis
/
ln.perfdb
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ln.perfdb updates

master
Harald Wolff 2020-02-24 17:48:13 +01:00
parent e0693765fa
commit b13389af75
7 changed files with 430 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
IPerfFileProvider.cs
View File

@ -13,6 +13,6 @@ namespace ln.perfdb
{
public interface IPerfFileProvider
{
PerfFile GetPerfFile(string name);
NewPerfFile GetPerfFile(string name);
}
}

267
NewPerfFile.cs 100644
View File

@ -0,0 +1,267 @@
// /**
// * File: PerfFile.cs
// * Author: haraldwolff
// *
// * This file and it's content is copyrighted by the Author and / or copyright holder.
// * Any use wihtout proper permission is illegal and may lead to legal actions.
// *
// *
// **/
using System;
using System.IO;
using ln.logging;
using System.Runtime.Remoting.Messaging;
using System.Text;
using ln.types.threads;
using System.Collections.Generic;
using System.IO.MemoryMappedFiles;
using ln.types;
using System.Linq;
namespace ln.perfdb
{
/**
* PerfFile Format Specification
*
* File Header:
*
* 0x0000 4 MAGIC
* 0x0004 4 Version
* 0x0008 4 nSections
* 0x000C 4 RESERVE
* 0x0010 16 Section specification #0
*[0x0020 16 Section specification #1]
*[...]
* Section Data #0
*[Section Data #1]
*
* Section specification format:
* 0x0000 4 Flags
* 0x0004 4 nRecords
* 0x0008 4 tWindow
* 0x000C 4 Aggregation Method
*
**/
public class NewPerfFile : IDisposable
{
public static bool DEBUG = false;
public readonly static ulong Magic1 = 0xBEAF8888BEAF8888L;
public readonly static ulong Magic2 = 0xBAD80000BAD90000L;
public readonly int SecondsPerHour = 3600;
public readonly int SecondsPerDay = 3600 * 24;
public readonly int SecondsPerWeek = 3600 * 24 * 7;
public String FileName { get; }
public FileStream FileStream { get; private set; }
public TimeWindow[] TimeWindows => twi.Select((arg) => arg.TimeWindow).ToArray();
TWImpl[] twi;
FileStream fileStream;
MemoryMappedFile mappedFile;
MemoryMappedViewAccessor mappedViewAccessor;
public NewPerfFile(String filename)
{
FileName = filename;
fileStream = new FileStream(filename, FileMode.Open);
if ((fileStream.ReadULong() != Magic1) || (fileStream.ReadULong() != Magic2))
throw new FormatException();
twi = new TWImpl[fileStream.ReadInteger()];
for (int n=0;n<twi.Length;n++)
{
twi[n] = new TWImpl(this, new TimeWindow(fileStream.ReadInteger(), fileStream.ReadInteger()));
if (n > 0)
twi[n - 1].Next = twi[n];
}
mappedFile = MemoryMappedFile.CreateFromFile(filename);
mappedViewAccessor = mappedFile.CreateViewAccessor(4096, 0);
Scan();
}
private void Scan()
{
foreach (TWImpl t in twi)
t.Scan();
}
public static NewPerfFile Create(string filename, TimeWindow[] timeWindows)
{
if (timeWindows.Length > 32)
throw new ArgumentOutOfRangeException(nameof(timeWindows));
using (FileStream fileStream = new FileStream(filename, FileMode.CreateNew))
{
long targetFileSize = 4096;
fileStream.WriteULong(Magic1);
fileStream.WriteULong(Magic2);
fileStream.WriteInteger(timeWindows.Length);
foreach (TimeWindow timeWindow in timeWindows)
{
fileStream.WriteInteger(timeWindow.NumRecords);
fileStream.WriteInteger(timeWindow.Interval);
targetFileSize += (timeWindow.NumRecords * 16);
}
fileStream.Position = targetFileSize - 1;
fileStream.WriteByte(0);
fileStream.Close();
}
return new NewPerfFile(filename);
}
public void Write(double value) => Write(DateTimeOffset.Now.ToUnixTimeSeconds(), value);
public void Write(long timestamp,double value) => twi[0].WriteValue(timestamp, value);
public void Close()
{
mappedViewAccessor?.Dispose();
mappedFile?.Dispose();
mappedViewAccessor = null;
mappedFile = null;
}
public void Dispose()
{
Close();
}
class TWImpl
{
public TimeWindow TimeWindow;
public NewPerfFile PerfFile;
TWImpl next;
public TWImpl Next
{
get => next;
set
{
if (value != null)
{
int asize = value.TimeWindow.Interval / TimeWindow.Interval;
if ((asize * TimeWindow.Interval) != value.TimeWindow.Interval)
throw new ArgumentOutOfRangeException(nameof(value.TimeWindow.Interval));
value.Offset = offset + TimeWindow.ByteSize;
aggregateSize = asize;
}
next = value;
}
}
long offset = 0;
public long Offset
{
get => offset;
set
{
offset = value;
if (next != null)
next.Offset = offset + TimeWindow.ByteSize;
}
}
int lastPosition = 0;
int aggregateSize = 1;
public TWImpl(NewPerfFile newPerfFile,TimeWindow timeWindow)
{
PerfFile = newPerfFile;
TimeWindow = timeWindow;
}
public void Scan()
{
long maxTimeStamp = 0;
for (int p = 0; p < TimeWindow.NumRecords; p++)
{
ReadPosition(p, out PerfValue perfValue);
if (perfValue.TimeStamp > maxTimeStamp)
{
lastPosition = (int)(perfValue.TimeStamp / TimeWindow.Interval);
maxTimeStamp = perfValue.TimeStamp;
}
}
}
public void ReadPosition(int position, out PerfValue perfValue)
{
position %= TimeWindow.NumRecords;
PerfFile.mappedViewAccessor.Read<PerfValue>(offset + (position * 16), out perfValue);
}
public void WritePosition(int position, PerfValue perfValue)
{
position %= TimeWindow.NumRecords;
PerfFile.mappedViewAccessor.Write<PerfValue>(offset + (position * 16), ref perfValue);
}
public void WriteValue(long timestamp,double value)
{
int position = (int)(timestamp / TimeWindow.Interval);
if (next != null)
{
// ToDo: Fix aggregation window
Aggregate(lastPosition, position);
}
}
public void Aggregate(int fromPosition,int toPosition)
{
if (fromPosition > toPosition)
throw new ArgumentOutOfRangeException(nameof(fromPosition));
int agStartIdx = fromPosition % aggregateSize;
int agStart = fromPosition - agStartIdx;
if (agStartIdx > 0)
agStart += aggregateSize;
int agEndIndex = toPosition % aggregateSize;
int agEnd = toPosition + aggregateSize - agEndIndex;
//int lastloop = Math.DivRem(fromPosition, TimeWindow.NumRecords, out int lastLoopIdx);
//int loop = Math.DivRem(toPosition, TimeWindow.NumRecords, out int loopIdx);
//int agStart = (lastLoopIdx + aggregateSize - 1) % aggregateSize;
//int agEnd = (loopIdx + aggregateSize - 1) & aggregateSize;
}
}
}
}
/*
//int pos = timeWindows[0].GetValuePositionFromTimestamp(timestamp);
//if (pos < tws[0].lastPosition)
// return;
//PerfValue perfValue = new PerfValue(0, 0);
//for (int iPos = tws[0].lastPosition;iPos != pos;iPos = (iPos + 1) % timeWindows[0].NumRecords)
//{
// mappedViewAccessor.Write<PerfValue>(tws[0].offset + (iPos * 16), ref perfValue);
//}
//perfValue.TimeStamp = timestamp;
//perfValue.Value = value;
//mappedViewAccessor.Write<PerfValue>(tws[0].offset + (pos * 16), ref perfValue);
*/

74
PerfDB.cs 100644
View File

@ -0,0 +1,74 @@
using System;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using System.IO;
using ln.types;
namespace ln.perfdb
{
public class PerfDB
{
public TimeWindow[] DefaultTimeWindowDefinition { get; set; } = new TimeWindow[]
{
new TimeWindow(TimeSpan.FromDays(28), 60),
new TimeWindow(TimeSpan.FromDays(56), 300),
new TimeWindow(TimeSpan.FromDays(84), 900),
new TimeWindow(TimeSpan.FromDays(168), 1800),
new TimeWindow(TimeSpan.FromDays(750), 3600)
};
public DirectoryInfo BasePath { get; }
Dictionary<Regex, TimeWindow[]> timeWindowDefinitions = new Dictionary<Regex, TimeWindow[]>();
Dictionary<string, NewPerfFile> perfFiles = new Dictionary<string, NewPerfFile>();
public PerfDB(String path)
{
BasePath = new DirectoryInfo(path);
}
public void SetTimeWindowDefintion(String regex, TimeWindow[] timeWindows) => SetTimeWindowDefintion(new Regex(regex), timeWindows);
public void SetTimeWindowDefintion(Regex regex,TimeWindow[] timeWindows)
{
timeWindowDefinitions[regex] = timeWindows;
}
public TimeWindow[] LookupTimeWindowDefinition(string perfName)
{
foreach (Regex regex in timeWindowDefinitions.Keys)
{
if (regex.IsMatch(perfName))
return timeWindowDefinitions[regex];
}
return DefaultTimeWindowDefinition;
}
public NewPerfFile GetPerfFile(string perfName)
{
string[] perfPath = perfName.Split('/');
lock (this)
{
if (!perfFiles.ContainsKey(perfName))
{
string perfDirectory = Path.Combine(BasePath.FullName, Path.Combine(perfPath.Slice(0, -1)));
if (!Directory.Exists(perfDirectory))
{
Directory.CreateDirectory(perfDirectory);
}
string perfFilePath = Path.Combine(BasePath.FullName, Path.Combine(perfPath));
if (!File.Exists(perfFilePath))
{
TimeWindow[] timeWindows = LookupTimeWindowDefinition(perfName);
perfFiles[perfName] = NewPerfFile.Create(perfFilePath, timeWindows);
}
else
{
perfFiles[perfName] = new NewPerfFile(perfFilePath);
}
}
}
return perfFiles[perfName];
}
}
}

34
PerfValue.cs 100644
View File

@ -0,0 +1,34 @@
// /**
// * File: PerfValue.cs
// * Author: haraldwolff
// *
// * This file and it's content is copyrighted by the Author and / or copyright holder.
// * Any use wihtout proper permission is illegal and may lead to legal actions.
// *
// *
// **/
using System;
namespace ln.perfdb
{
public struct PerfValue
{
public long TimeStamp;
public double Value;
public PerfValue(long timeStamp, double value)
{
TimeStamp = timeStamp;
Value = value;
}
public PerfValue(double value)
{
TimeStamp = DateTimeOffset.Now.ToUnixTimeSeconds();
Value = value;
}
public override string ToString()
{
return String.Format("Timestamp: {0,12} {1}",TimeStamp,Value);
}
}
}

30
TimeWindow.cs 100644
View File

@ -0,0 +1,30 @@
using System;
namespace ln.perfdb
{
public class TimeWindow
{
public int NumRecords { get; private set; }
public int Interval { get; private set; }
public TimeWindow(int numRecords, int interval)
{
NumRecords = numRecords;
Interval = interval;
}
public TimeWindow(TimeSpan timeSpan, int interval)
{
Interval = interval;
TimeSpan = timeSpan;
}
public TimeSpan TimeSpan {
get => TimeSpan.FromSeconds(NumRecords * Interval);
private set => NumRecords = (int)(value.TotalSeconds / Interval);
}
public int GetValuePositionFromTimestamp(long timestamp) => ((int)(timestamp / Interval));
public int GetValueIndexFromTimestamp(long timestamp) => ((int)(timestamp / Interval)) % NumRecords;
public long ByteSize => NumRecords * 16;
}
}

9
ln.perfdb.csproj
View File

@ -28,6 +28,9 @@
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="nunit.framework">
<Package>nunit</Package>
</Reference>
</ItemGroup>
<ItemGroup>
<Compile Include="Properties\AssemblyInfo.cs" />
@ -38,9 +41,15 @@
<Compile Include="storage\Aggregates.cs" />
<Compile Include="storage\PerfFile.PerfFileSection.cs" />
<Compile Include="IPerfFileProvider.cs" />
<Compile Include="NewPerfFile.cs" />
<Compile Include="TimeWindow.cs" />
<Compile Include="PerfDB.cs" />
<Compile Include="test\PerfDBTests.cs" />
<Compile Include="PerfValue.cs" />
</ItemGroup>
<ItemGroup>
<Folder Include="storage\" />
<Folder Include="test\" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\ln.logging\ln.logging.csproj">

15
test/PerfDBTests.cs 100644
View File

@ -0,0 +1,15 @@
using NUnit.Framework;
using System;
namespace ln.perfdb.test
{
[TestFixture()]
public class PerfDBTests
{
[Test()]
public void TestCase()
{
PerfDB perfDB = new PerfDB(".");
NewPerfFile npf = perfDB.GetPerfFile("test/ind0");
}
}
}