diff --git a/SkyEntities.cs b/SkyEntities.cs index b954a77..9080e1d 100644 --- a/SkyEntities.cs +++ b/SkyEntities.cs @@ -11,7 +11,6 @@ using System; using ln.types.odb; using ln.skyscanner.entities; using System.IO; -using LiteDB; using ln.http.resources; using System.Linq; diff --git a/checks/CheckJob.cs b/checks/CheckJob.cs index a0c62e3..819dbdd 100644 --- a/checks/CheckJob.cs +++ b/checks/CheckJob.cs @@ -14,6 +14,7 @@ using ln.logging; using Newtonsoft.Json; using ln.types.odb; using System.Linq; +using System.Collections.Generic; namespace ln.skyscanner.checks { public class CheckJob : PoolJob @@ -21,10 +22,19 @@ namespace ln.skyscanner.checks [JsonIgnore] public Node Node { get; } + Dictionary checkStates = new Dictionary(); + public CheckJob(Node node) { Name = String.Format("Interval check: {0} [{1}]",node.UniqueIdentity,node.PrimaryIP.ToString()); Node = node; + + Query stateQuery = Query.Equals("UniqueNodeIdentifier", Node.UniqueIdentity); + SkyCheckState[] skyCheckStates = SkyScanner.Instance.Checker.checkStates.Select(stateQuery).ToArray(); + foreach (SkyCheckState checkState in skyCheckStates) + { + checkStates.Add(checkState.CheckName, checkState); + } } public override void RunJob() @@ -38,17 +48,18 @@ namespace ln.skyscanner.checks if (checks[n].IsValid(Node)) { - Query stateQuery = Query.AND( - Query.Equals("CheckName",checks[n].Name), - Query.Equals("UniqueNodeIdentifier", Node.UniqueIdentity) - ); - SkyCheckState checkState = SkyScanner.Instance.Checker.CheckStates.Select(stateQuery).FirstOrDefault(); - try { + SkyCheckState checkState = checkStates.ContainsKey(checks[n].Name) ? checkStates[checks[n].Name] : null; + + if (checkState != null) + { + checkState.LastCheckTime = DateTime.Now; + } + checks[n].Check(SkyScanner.Instance.Checker, ref checkState, Node); if (checkState != null) - SkyScanner.Instance.Checker.CheckStates.Upsert(checkState); + SkyScanner.Instance.Checker.Save(checkState); } catch (Exception e) { diff --git a/checks/Hostalive.cs b/checks/Hostalive.cs index 25ac580..08add17 100644 --- a/checks/Hostalive.cs +++ b/checks/Hostalive.cs @@ -31,6 +31,13 @@ namespace ln.skyscanner.checks public override void Check(SkyChecker skyChecker,ref SkyCheckState checkState,Node node) { + if (checkState == null) + { + checkState = new SkyCheckState(this, node); + checkState.WarnLower = 0.6; + checkState.CritLower = 0.3; + } + long roundTripTime = 0; int success = 0; int n; @@ -53,6 +60,7 @@ namespace ln.skyscanner.checks roundTripTime /= success; skyChecker.WritePerfValue(this, "rta", node, roundTripTime); } + SetState(checkState, fSuccess); } } } diff --git a/checks/SkyCheck.cs b/checks/SkyCheck.cs index ee42d79..9fee775 100644 --- a/checks/SkyCheck.cs +++ b/checks/SkyCheck.cs @@ -34,6 +34,20 @@ namespace ln.skyscanner.checks public abstract bool IsValid(Node node); public abstract void Check(SkyChecker skyChecker,ref SkyCheckState checkState,Node node); + public virtual void SetState(SkyCheckState checkState,double value) + { + if ((value < checkState.CritLower) || (value > checkState.CritUpper)) + { + checkState.CheckState = CheckState.CRITICAL; + } else if ((value < checkState.WarnLower) || (value > checkState.WarnUpper)) + { + checkState.CheckState = CheckState.WARN; + } + else + { + checkState.CheckState = CheckState.OK; + } + } static SkyCheck() { diff --git a/checks/SkyCheckState.cs b/checks/SkyCheckState.cs index 2728595..3b2f9e8 100644 --- a/checks/SkyCheckState.cs +++ b/checks/SkyCheckState.cs @@ -9,12 +9,19 @@ // **/ using System; using ln.skyscanner.entities; +using ln.types.odb; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; namespace ln.skyscanner.checks { + [JsonConverter(typeof(StringEnumConverter))] public enum CheckState { OK, WARN, CRITICAL, FAIL, ERROR } public class SkyCheckState { + [DocumentID] + public Guid ID = Guid.NewGuid(); + public readonly String CheckName; public readonly String UniqueNodeIdentifier; @@ -25,6 +32,8 @@ namespace ln.skyscanner.checks public double CritLower { get; set; } = Double.MinValue; public double CritUpper { get; set; } = Double.MaxValue; + public CheckState CheckState { get; set; } + protected SkyCheckState() { } diff --git a/checks/SkyChecker.cs b/checks/SkyChecker.cs index 1eeff1c..fb43e8a 100644 --- a/checks/SkyChecker.cs +++ b/checks/SkyChecker.cs @@ -19,6 +19,7 @@ using ln.logging; using ln.snmp; using ln.http.resources; using ln.types.odb; +using System.Collections.Generic; namespace ln.skyscanner.checks { public class SkyChecker @@ -29,6 +30,9 @@ namespace ln.skyscanner.checks public PoolJob[] CurrentJobs => checkPool.CurrentPoolJobs; [Callable] public PoolJob[] QueuedJobs => checkPool.QueuedJobs; + [Callable] + public SkyCheckState[] CheckStates => checkStates.ToArray(); + [Callable] public string[] PerfNames => perfFiles.Keys.ToArray(); @@ -44,7 +48,9 @@ namespace ln.skyscanner.checks bool stopping; ODB odb; - public ODBCollection CheckStates { get; private set; } + public ODBCollection checkStates { get; private set; } + + List saveQueue = new List(); public SkyChecker() { @@ -54,9 +60,9 @@ namespace ln.skyscanner.checks SNMPEngine.Timeout = 4000; odb = new ODB(Path.Combine(BasePath)); - CheckStates = odb.GetCollection(); - CheckStates.EnsureIndex("CheckName"); - CheckStates.EnsureIndex("UniqueNodeIdentifier"); + checkStates = odb.GetCollection(); + checkStates.EnsureIndex("CheckName"); + checkStates.EnsureIndex("UniqueNodeIdentifier"); } public void Start() @@ -64,7 +70,9 @@ namespace ln.skyscanner.checks if ((threadScheduler != null) && (threadScheduler.IsAlive)) throw new NotSupportedException("SkyChecker.scheduler() already running"); - checkPool.SetPoolSize(128); + checkPool.SetPoolSize(256); + + Thread.Sleep(2500); threadScheduler = new Thread(scheduler); threadScheduler.Start(); @@ -154,14 +162,28 @@ namespace ln.skyscanner.checks { while (!stopping) { - int n = 0; + List checkJobs = new List(); + + Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): scheduler save CheckStates"); + + lock (saveQueue) + { + foreach (SkyCheckState checkState in saveQueue) + { + SkyScanner.Instance.Checker.checkStates.Upsert(checkState); + } + saveQueue.Clear(); + } + + Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): scheduler starts"); + foreach (Node node in SkyScanner.Instance.Entities.GlobalNetwork.Nodes) { CheckJob checkJob = new CheckJob(node); - if (checkPool.Enqueue(checkJob)) - n++; + checkJobs.Add(checkJob); } - Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): scheduled {0} checks", n); + Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): prepared {0} checks", checkJobs.Count); + Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): scheduled {0} checks", checkPool.Enqueue(checkJobs)); while ((nextMinute - DateTimeOffset.Now.ToUnixTimeMilliseconds()) < 0) nextMinute += 60000; @@ -176,6 +198,14 @@ namespace ln.skyscanner.checks } + public void Save(SkyCheckState checkState) + { + lock (saveQueue) + { + saveQueue.Add(checkState); + } + } + public ComponentState State { diff --git a/checks/Ubiquity.cs b/checks/Ubiquity.cs index d68c8cb..c994f6a 100644 --- a/checks/Ubiquity.cs +++ b/checks/Ubiquity.cs @@ -56,6 +56,7 @@ namespace ln.skyscanner.checks } } + checkState.CheckState = CheckState.OK; } public override bool IsValid(Node node) diff --git a/crawl/CrawledHost.cs b/crawl/CrawledHost.cs index aed96d2..a127147 100644 --- a/crawl/CrawledHost.cs +++ b/crawl/CrawledHost.cs @@ -13,7 +13,6 @@ using ln.types; using System.Net; using System.Linq; using ln.snmp.rfc1213; -using LiteDB; using ln.types.odb; using ln.types.net; diff --git a/entities/GlobalNetwork.cs b/entities/GlobalNetwork.cs index f1d4b9a..1f83a94 100644 --- a/entities/GlobalNetwork.cs +++ b/entities/GlobalNetwork.cs @@ -26,7 +26,7 @@ namespace ln.skyscanner.entities public SkyEntities SkyEntities { get; private set; } public IEnumerable Subnets => SkyEntities.subnetCollection; - public IEnumerable Nodes => SkyEntities.nodeCollection; + public IEnumerable Nodes => SkyEntities.nodeCollection.ToArray(); public HopMap DefaultHopMap { get; private set; } = new HopMap(); diff --git a/entities/HopMap.cs b/entities/HopMap.cs index 28a43eb..0bfcfb6 100644 --- a/entities/HopMap.cs +++ b/entities/HopMap.cs @@ -9,15 +9,11 @@ // **/ using System; using System.Collections.Generic; -using System.Collections; -using Castle.Core.Logging; using ln.types.odb; using System.Linq; using ln.types.btree; -using Castle.Components.DictionaryAdapter.Xml; using Newtonsoft.Json; using ln.types.net; -using System.Net.Sockets; namespace ln.skyscanner.entities { public class HopMap : Persistent diff --git a/entities/NetworkInterface.cs b/entities/NetworkInterface.cs index 6fd7407..6eacda3 100644 --- a/entities/NetworkInterface.cs +++ b/entities/NetworkInterface.cs @@ -13,7 +13,6 @@ using ln.types; using ln.skyscanner.crawl; using ln.types.odb; using Newtonsoft.Json; -using LiteDB; using ln.logging; using ln.types.net; namespace ln.skyscanner.entities diff --git a/entities/Node.cs b/entities/Node.cs index d1240d3..9dd9032 100644 --- a/entities/Node.cs +++ b/entities/Node.cs @@ -13,7 +13,6 @@ using System.Collections.Generic; using System.Linq; using ln.types.odb; using Newtonsoft.Json; -using LiteDB; using ln.types.threads; using ln.types.net; namespace ln.skyscanner.entities diff --git a/ln.skyscanner.csproj b/ln.skyscanner.csproj index 7b6cb00..56ea8cb 100644 --- a/ln.skyscanner.csproj +++ b/ln.skyscanner.csproj @@ -36,13 +36,7 @@ ..\packages\Newtonsoft.Json.12.0.1\lib\net45\Newtonsoft.Json.dll - - ..\packages\Castle.Core.4.3.1\lib\net45\Castle.Core.dll - - - ..\packages\LiteDB.4.1.4\lib\net40\LiteDB.dll - @@ -166,6 +160,9 @@ PreserveNewest + + PreserveNewest + diff --git a/packages.config b/packages.config index b1b0b2b..78afeed 100644 --- a/packages.config +++ b/packages.config @@ -1,7 +1,5 @@  - - \ No newline at end of file diff --git a/templates/static/checks/checkstates.html b/templates/static/checks/checkstates.html new file mode 100644 index 0000000..d4f8307 --- /dev/null +++ b/templates/static/checks/checkstates.html @@ -0,0 +1,42 @@ +<%frame "frame.html"%> + +

Check Stati

+
+ +
+
+
+
+
+
+
+ + diff --git a/templates/static/checks/index.html b/templates/static/checks/index.html index 7b6823a..aad08f2 100644 --- a/templates/static/checks/index.html +++ b/templates/static/checks/index.html @@ -21,6 +21,7 @@