From 895231b820572d8ebf3e8db343c5257231a324a8 Mon Sep 17 00:00:00 2001 From: Harald Wolff Date: Thu, 29 Aug 2019 13:14:52 +0200 Subject: [PATCH] Broken WIP --- Program.cs | 30 +- SkyEntities.cs | 131 -- SkyScanner.cs | 18 +- checks/APC.cs | 27 +- checks/CheckJob.cs | 68 +- checks/Hostalive.cs | 8 +- checks/Mimosa.cs | 8 +- checks/SikluCheck.cs | 11 +- checks/SkyCheck.cs | 2 +- checks/SkyCheckState.cs | 53 +- checks/SkyChecker.cs | 210 --- checks/Ubiquiti.cs | 20 +- crawl/CrawledHost.cs | 6 - crawl/CrawledSubnet.cs | 7 +- crawl/Crawler.cs | 16 +- crawl/service/CrawlService.cs | 3 +- crawl/service/SNMP.cs | 468 +++---- crawl/service/SSH.cs | 216 +-- crawl/service/TCP.cs | 146 +- crawl/service/Ubiquity.cs | 107 +- entities/ConfiguredIP.cs | 3 +- entities/GlobalNetwork.cs | 512 ++++--- entities/HopMap.cs | 300 ++-- entities/L2Segment.cs | 2 - entities/NetworkInterface.cs | 6 - entities/Node.cs | 50 +- entities/PointOfPresence.cs | 11 +- entities/Subnet.cs | 6 +- http/CheckerApi.cs | 130 -- http/CrawlerApi.cs | 257 ++-- http/NetworkApi.cs | 52 +- http/SkyScannerHttpApi.cs | 5 - http/SkyScannerHttpApplication.cs | 5 +- ln.skyscanner.csproj | 36 +- {checks => perfvalue}/PerformanceValue.cs | 17 +- services/CheckService.cs | 72 +- services/EntityService.cs | 128 +- services/PerformanceValueService.cs | 117 ++ templates/static/checks/issues2.html | 324 ++--- templates/static/checks/node.html | 123 +- templates/static/css/style.css | 37 + .../static/dist/moment-timezone-with-data.js | 1224 +++++++++++++++++ templates/static/frame.html | 1 + templates/static/system/index.html | 16 +- 44 files changed, 2955 insertions(+), 2034 deletions(-) delete mode 100644 SkyEntities.cs delete mode 100644 checks/SkyChecker.cs delete mode 100644 http/CheckerApi.cs rename {checks => perfvalue}/PerformanceValue.cs (80%) create mode 100644 services/PerformanceValueService.cs create mode 100644 templates/static/dist/moment-timezone-with-data.js diff --git a/Program.cs b/Program.cs index dac9e7c..911940f 100644 --- a/Program.cs +++ b/Program.cs @@ -7,44 +7,16 @@ // * // * // **/ -using System; -using System.Collections.Generic; -using ln.snmp; using ln.logging; -using ln.skyscanner.entities; -using ln.types.threads; -using ln.types.net; -using ln.types.odb; -using ln.types.odb.index; -using ln.types.odb.values; -using ln.skyscanner.checks; -using System.Linq; -using ln.skyscanner.import.skytron; namespace ln.skyscanner { class MainClass { - - public static void Main(string[] args) { - FileLogger fileLogger = new FileLogger("skyscanner.log"); - fileLogger.MaxLogLevel = LogLevel.INFO; - Logger.Default.Backends.Add(fileLogger); - Logger.ConsoleLogger.MaxLogLevel = LogLevel.INFO; - - SkyScanner skyScanner = new SkyScanner(); - - Initialize(); - - skyScanner.Start(args); - } - - private static void Initialize() - { -// SNMPEngine.DefaultEngine.Timeout = 3500; + new SkyScanner().Start(args); } } diff --git a/SkyEntities.cs b/SkyEntities.cs deleted file mode 100644 index a4e4f1e..0000000 --- a/SkyEntities.cs +++ /dev/null @@ -1,131 +0,0 @@ -// /** -// * File: SkyEntities.cs -// * Author: haraldwolff -// * -// * This file and it's content is copyrighted by the Author and / or copyright holder. -// * Any use wihtout proper permission is illegal and may lead to legal actions. -// * -// * -// **/ -using System; -using ln.types.odb; -using ln.skyscanner.entities; -using System.IO; -using ln.http.resources; -using System.Linq; -using ln.types.odb.mapped; -using ln.types.net; -using ln.skyscanner.crawl; -using ln.skyscanner.checks; -using ln.logging; -using System.Collections.Generic; - -namespace ln.skyscanner -{ - public class SkyEntities - { - public SkyScanner SkyScanner { get; } - public string BasePath => Path.Combine(SkyScanner.BasePath, "entities"); - - public GlobalNetwork GlobalNetwork { get; private set; } - - - public ODB ODB { get; private set; } - - public ODBCollection NodeCollection { get; private set; } - public ODBCollection SubnetCollection { get; private set; } - - public ODBCollection PointOfPresenceCollection { get; private set; } - public ODBCollection L2SegmentCollection { get; private set; } - - public ODBCollection CrawledHosts { get; private set; } - public ODBCollection CrawledSubnets { get; private set; } - - public ODBCollection BlockedNetworks { get; private set; } - - public ODBCollection SkyCheckStates { get; private set; } - - public SkyEntities(SkyScanner skyScanner) - { - SkyScanner = skyScanner; - - Logging.Log(LogLevel.INFO, "SkyEntities: initializing"); - - ODB = new ODB(BasePath); - - NodeCollection = ODB.GetCollection(); - NodeCollection.EnableStrongCache(true); - NodeCollection.EnsureIndex("uniqueIdentity", true); - NodeCollection.EnsureIndeces( - "PrimaryIP", - "Interfaces[].ConfiguredIPs[].IP", - "Interfaces[].ConfiguredIPs[].Network" - ); - - Network4 n192 = Network4.Parse("192.168.0.0/16"); - - /* Preload all nodes to increase load speed*/ - foreach (Node node in NodeCollection.ToArray()) - { - //if (n192.Contains(node.PrimaryIP)) - //NodeCollection.Delete(node); - } - - SubnetCollection = ODB.GetCollection(); - SubnetCollection.EnsureIndeces("Network"); - - PointOfPresenceCollection = ODB.GetCollection(); - PointOfPresenceCollection.EnsureIndeces("ForeignName"); - - L2SegmentCollection = ODB.GetCollection(); - L2SegmentCollection.EnsureIndeces("Network","PoPs"); - - CrawledHosts = ODB.GetCollection(); - CrawledHosts.EnsureIndeces("PrimaryIP","IPAddresses[]"); - - CrawledSubnets = ODB.GetCollection(); - BlockedNetworks = ODB.GetCollection("blockedNetworks"); - - SkyCheckStates = ODB.GetCollection(); - SkyCheckStates.EnableStrongCache(true); - SkyCheckStates.EnsureIndeces("Node"); - SkyCheckStates.EnsureUniqueness("Node", "CheckName"); - - //foreach (SkyCheckState checkState in SkyCheckStates.ToArray()) - //{ - // if (object.ReferenceEquals(checkState.Node, null)) - // SkyCheckStates.Delete(checkState); - //} - - Logging.Log(LogLevel.INFO, "SkyEntities: initialized"); - - GlobalNetwork = new GlobalNetwork(); - } - - public PerformanceValue LookupPerformanceValue(String perfName) - { - string[] resourcePath = perfName.Split('/'); - - Node node = NodeCollection.Query("uniqueIdentity", resourcePath[0]).FirstOrDefault(); - if (node == null) - throw new KeyNotFoundException(); - - SkyCheckState skyCheckState = SkyCheckStates.Query( - Query.AND( - Query.Equals("Node", node.ID), - Query.Equals("CheckName", resourcePath[1]) - ) - ).FirstOrDefault(); - if (skyCheckState == null) - throw new KeyNotFoundException(); - - foreach (PerformanceValue performanceValue in skyCheckState.PerformanceValues) - if (performanceValue.PerfName.Equals(perfName)) - return performanceValue; - - throw new KeyNotFoundException(); - } - - - } -} diff --git a/SkyScanner.cs b/SkyScanner.cs index 877b0f8..65c4c13 100644 --- a/SkyScanner.cs +++ b/SkyScanner.cs @@ -4,7 +4,6 @@ using ln.http; using System.Net; using ln.skyscanner.http; using System.IO; -using ln.skyscanner.crawl; using ln.types.threads; using ln.skyscanner.checks; using ln.skyscanner.import.skytron; @@ -27,17 +26,8 @@ namespace ln.skyscanner public static SkyScanner Instance { get; private set; } public String BasePath { get; private set; } - public Pool ConveniencePool { get; } - public HTTPServer HTTPServer { get; private set; } - public Crawler Crawler { get; private set; } - public SkyEntities Entities { get; private set; } - public SkyChecker Checker { get; private set; } - - public bool OptionDisableChecker { get; private set; } - public String OptionCrawlVendor { get; private set; } - public SkyScanner() { if (Instance != null) @@ -59,7 +49,6 @@ namespace ln.skyscanner BasePath = Arguments["base-path"].Value; Logging.Log(LogLevel.INFO, "SkyScanner: BasePath={0}", BasePath); - ConveniencePool = new Pool(); } @@ -67,6 +56,10 @@ namespace ln.skyscanner { base.PrepareStart(); + FileLogger fileLogger = new FileLogger("skyscanner.log"); + fileLogger.MaxLogLevel = (LogLevel)Enum.Parse(typeof(LogLevel), Arguments["log-level"].Value); + Logger.Default.Backends.Add(fileLogger); + if (Arguments["import-skytron"].IsSet) { SkytronImport si = new SkytronImport(Arguments["import-skytron"].Value); @@ -104,6 +97,7 @@ namespace ln.skyscanner ServiceContainer.Add(ServiceDefinition.From(true)); ServiceContainer.Add(ServiceDefinition.From(true)); ServiceContainer.Add(ServiceDefinition.From(true)); + ServiceContainer.Add(ServiceDefinition.From(true)); } @@ -254,7 +248,7 @@ namespace ln.skyscanner else { Logging.Log(LogLevel.INFO, "DebugCheck(): Node: {0}",JSONObject.From(node).ToString()); - CheckJob checkJob = new CheckJob(node); + CheckJob checkJob = new CheckJob(null, node); Logging.Log(LogLevel.INFO, "Prepare..."); checkJob.Prepare(); diff --git a/checks/APC.cs b/checks/APC.cs index 4b09584..d5cd342 100644 --- a/checks/APC.cs +++ b/checks/APC.cs @@ -7,6 +7,7 @@ using ln.snmp.types; using ln.logging; using Microsoft.Win32.SafeHandles; using Renci.SshNet.Messages.Connection; +using ln.skyscanner.services; namespace ln.skyscanner.checks { @@ -18,11 +19,11 @@ namespace ln.skyscanner.checks } - public override void Check(SkyChecker skyChecker,ref SkyCheckState checkState,Node node) + public override void Check(CheckService checkService, ref SkyCheckState checkState,Node node) { foreach (URI snmpUri in node.FindURIs("snmp")) { - using (SnmpInterface snmp = SnmpInterface.FromURI(snmpUri,skyChecker.SNMPEngine)) + using (SnmpInterface snmp = SnmpInterface.FromURI(snmpUri,checkService.SNMPEngine)) { try { @@ -51,29 +52,29 @@ namespace ln.skyscanner.checks double voltage = (double)((Integer)(input[1].Items[1])).LongValue; double current = (double)((Integer)(input[2].Items[1])).LongValue; - checkState.WritePerformanceValue(String.Format("ups_input_{0}_frequency", n), frequency, wLower: 48, wUpper: 52, cLower: 45, cUpper: 55); - checkState.WritePerformanceValue(String.Format("ups_input_{0}_voltage", n), voltage, wLower: 218.5, wUpper: 238, cLower: 212, cUpper: 245); - checkState.WritePerformanceValue(String.Format("ups_input_{0}_current", n), current); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_input_{0}_frequency", n), frequency, wLower: 48, wUpper: 52, cLower: 45, cUpper: 55); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_input_{0}_voltage", n), voltage, wLower: 218.5, wUpper: 238, cLower: 212, cUpper: 245); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_input_{0}_current", n), current); n++; } n = 0; foreach (Sequence[] output in outputs) { - checkState.WritePerformanceValue(String.Format("ups_output_{0}_voltage", n), (double)((Integer)(output[0].Items[1])).LongValue, wLower: 218.5, wUpper: 238, cLower: 212, cUpper: 245); - checkState.WritePerformanceValue(String.Format("ups_output_{0}_current", n), (double)((Integer)(output[1].Items[1])).LongValue / 10.0); - checkState.WritePerformanceValue(String.Format("ups_output_{0}_load", n), (double)((Integer)(output[2].Items[1])).LongValue, wUpper: 50, cUpper: 75); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_output_{0}_voltage", n), (double)((Integer)(output[0].Items[1])).LongValue, wLower: 218.5, wUpper: 238, cLower: 212, cUpper: 245); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_output_{0}_current", n), (double)((Integer)(output[1].Items[1])).LongValue / 10.0); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_output_{0}_load", n), (double)((Integer)(output[2].Items[1])).LongValue, wUpper: 50, cUpper: 75); n++; } n = 0; foreach (Sequence[] battery in batteries) { - checkState.WritePerformanceValue(String.Format("ups_battery_{0}_status", n), (double)((Integer)(battery[0].Items[1])).LongValue); - checkState.WritePerformanceValue(String.Format("ups_battery_{0}_minutes_remain", n), (double)((Integer)(battery[1].Items[1])).LongValue, wLower: 20, cLower: 10 ); - checkState.WritePerformanceValue(String.Format("ups_battery_{0}_capacity", n), (double)((Integer)(battery[2].Items[1])).LongValue, wLower: 75, cLower: 50); - checkState.WritePerformanceValue(String.Format("ups_battery_{0}_voltage", n), (double)((Integer)(battery[3].Items[1])).LongValue / 10.0); - checkState.WritePerformanceValue(String.Format("ups_battery_{0}_temperature", n), (double)((Integer)(battery[4].Items[1])).LongValue, wUpper: 40, cUpper: 60); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_battery_{0}_status", n), (double)((Integer)(battery[0].Items[1])).LongValue); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_battery_{0}_minutes_remain", n), (double)((Integer)(battery[1].Items[1])).LongValue, wLower: 20, cLower: 10 ); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_battery_{0}_capacity", n), (double)((Integer)(battery[2].Items[1])).LongValue, wLower: 75, cLower: 50); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_battery_{0}_voltage", n), (double)((Integer)(battery[3].Items[1])).LongValue / 10.0); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_battery_{0}_temperature", n), (double)((Integer)(battery[4].Items[1])).LongValue, wUpper: 40, cUpper: 60); n++; } diff --git a/checks/CheckJob.cs b/checks/CheckJob.cs index 6066513..760a519 100644 --- a/checks/CheckJob.cs +++ b/checks/CheckJob.cs @@ -11,62 +11,27 @@ using System; using ln.types.threads; using ln.skyscanner.entities; using ln.logging; -using ln.types.odb; -using System.Linq; -using System.Collections.Generic; +using ln.skyscanner.services; namespace ln.skyscanner.checks { public class CheckJob : PoolJob { + public CheckService CheckService { get; } public Node Node { get; } - public SkyCheckState[] CheckStates - { - get - { - return checkStates.Values.ToArray(); - } - set - { - checkStates.Clear(); - foreach (SkyCheckState checkState in value) - { - checkStates.Add(checkState.CheckName, checkState); - } - } - - } - - Dictionary checkStates = new Dictionary(); - - public CheckJob(Node node) + public CheckJob(CheckService checkService,Node node) { + CheckService = checkService; Name = String.Format("Interval check: {0} [{1}]",node.UniqueIdentity,node.PrimaryIP.ToString()); Node = node; } - public override void Prepare() - { - /* - Query stateQuery = Query.Equals("Node", Node.ID); - SkyCheckState[] skyCheckStates = SkyScanner.Instance.Entities.SkyCheckStates.Query(stateQuery).ToArray(); - foreach (SkyCheckState checkState in skyCheckStates) - { - checkStates.Add(checkState.CheckName, checkState); - } - foreach (SkyCheck check in SkyCheck.SkyChecks) - if (!checkStates.ContainsKey(check.Name) && check.IsValid(Node)) - { - checkStates.Add(check.Name, check.PrepareCheckState(SkyScanner.Instance.Checker, Node)); - SkyScanner.Instance.Entities.SkyCheckStates.Insert(checkStates[check.Name]); - } - */ - } - public override void RunJob() { SkyCheck[] checks = SkyCheck.SkyChecks; + long startTime = DateTimeOffset.Now.ToUnixTimeMilliseconds(); + for (int n=0;n true; public override bool IsValid(Node node) => node.PrimaryIP != null; - public override void Check(SkyChecker skyChecker,ref SkyCheckState checkState,Node node) + public override void Check(CheckService checkService, ref SkyCheckState checkState, Node node) { long roundTripTime = 0; int success = 0; @@ -47,11 +49,11 @@ namespace ln.skyscanner.checks float fSuccess = (float)success / (float)n; - checkState.WritePerformanceValue("replies", fSuccess, node.DeviceType == DeviceType.UNKNOWN ? Double.MinValue : 0.8, double.MaxValue, node.DeviceType == DeviceType.UNKNOWN ? Double.MinValue : 0.6, double.MaxValue); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, "replies", fSuccess, node.DeviceType == DeviceType.UNKNOWN ? Double.MinValue : 0.8, double.MaxValue, node.DeviceType == DeviceType.UNKNOWN ? Double.MinValue : 0.6, double.MaxValue); if (success > 0) { roundTripTime /= success; - checkState.WritePerformanceValue("rta", roundTripTime, 0, 80, 0, 140); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, "rta", roundTripTime, 0, 80, 0, 140); } foreach (PerformanceValue pv in checkState.PerformanceValues) diff --git a/checks/Mimosa.cs b/checks/Mimosa.cs index e034df2..f11904f 100644 --- a/checks/Mimosa.cs +++ b/checks/Mimosa.cs @@ -15,7 +15,7 @@ namespace ln.skyscanner.checks :base("mimosa") {} - public override void Check(SkyChecker skyChecker, ref SkyCheckState checkState, Node node) + public override void Check(CheckService checkService, ref SkyCheckState checkState, Node node) { MimosaCheckState mimosaCheckState = checkState as MimosaCheckState; @@ -23,7 +23,7 @@ namespace ln.skyscanner.checks { try { - using (SnmpInterface snmp = SnmpInterface.FromURI(snmpUri, skyChecker.SNMPEngine)) + using (SnmpInterface snmp = SnmpInterface.FromURI(snmpUri, checkService.SNMPEngine)) { Sequence[][] sigData = snmp.snmpWalk(new string[] { "1.3.6.1.4.1.43356.2.1.2.6.1.1.3", // Signal Level @@ -38,8 +38,8 @@ namespace ln.skyscanner.checks double sig = (sigData[n][0].Items[1] as Integer).LongValue / 10.0; double snr = (sigData[n][1].Items[1] as Integer).LongValue / 10.0; - mimosaCheckState.WritePerformanceValue(String.Format("ptp_rx_pwr_{0}",n), sig, -75.0, 0, -80.0, 0); - mimosaCheckState.WritePerformanceValue(String.Format("ptp_rx_snr_{0}",n), snr, 12, 99, 8, 99); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ptp_rx_pwr_{0}",n), sig, -75.0, 0, -80.0, 0); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ptp_rx_snr_{0}",n), snr, 12, 99, 8, 99); } diff --git a/checks/SikluCheck.cs b/checks/SikluCheck.cs index b175d6d..94843dd 100644 --- a/checks/SikluCheck.cs +++ b/checks/SikluCheck.cs @@ -4,6 +4,7 @@ using ln.logging; using ln.types; using ln.snmp; using ln.snmp.types; +using ln.skyscanner.services; namespace ln.skyscanner.checks { @@ -14,13 +15,13 @@ namespace ln.skyscanner.checks { } - public override void Check(SkyChecker skyChecker, ref SkyCheckState checkState, Node node) + public override void Check(CheckService checkService, ref SkyCheckState checkState, Node node) { foreach (URI snmpUri in node.FindURIs("snmp")) { try { - using (SnmpInterface snmp = SnmpInterface.FromURI(snmpUri, skyChecker.SNMPEngine)) + using (SnmpInterface snmp = SnmpInterface.FromURI(snmpUri, checkService.SNMPEngine)) { Sequence[][] ptp = snmp.snmpWalk(new string[] { "1.3.6.1.4.1.31926.2.1.1.19", // RX PWR @@ -31,9 +32,9 @@ namespace ln.skyscanner.checks int n = 0; foreach (Sequence[] row in ptp) { - checkState.WritePerformanceValue(String.Format("ptp_rx_pwr_{0}", n), (double)((Integer)(row[0].Items[1])).LongValue, -75.0, 0, -80.0, 0); - checkState.WritePerformanceValue(String.Format("ptp_tx_snr_{0}", n), (double)((Integer)(row[1].Items[1])).LongValue); - checkState.WritePerformanceValue(String.Format("ptp_tx_pwr_{0}", n), (double)((Integer)(row[2].Items[1])).LongValue); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ptp_rx_pwr_{0}", n), (double)((Integer)(row[0].Items[1])).LongValue, -75.0, 0, -80.0, 0); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ptp_tx_snr_{0}", n), (double)((Integer)(row[1].Items[1])).LongValue); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ptp_tx_pwr_{0}", n), (double)((Integer)(row[2].Items[1])).LongValue); /* long rxBytes = ((Integer)(row[4].Items[1])).LongValue; long txBytes = ((Integer)(row[5].Items[1])).LongValue; diff --git a/checks/SkyCheck.cs b/checks/SkyCheck.cs index 8d00919..3fe98f7 100644 --- a/checks/SkyCheck.cs +++ b/checks/SkyCheck.cs @@ -33,7 +33,7 @@ namespace ln.skyscanner.checks public virtual bool IsCritical => false; public abstract bool IsValid(Node node); - public abstract void Check(SkyChecker skyChecker,ref SkyCheckState checkState,Node node); + public abstract void Check(CheckService checkService,ref SkyCheckState checkState,Node node); public virtual SkyCheckState PrepareCheckState(CheckService skyChecker,Node node) { return new SkyCheckState(this, node); diff --git a/checks/SkyCheckState.cs b/checks/SkyCheckState.cs index 3fd3cc7..13951ab 100644 --- a/checks/SkyCheckState.cs +++ b/checks/SkyCheckState.cs @@ -9,8 +9,10 @@ // **/ using System; using ln.skyscanner.entities; -using ln.types.odb.attributes; using System.Collections.Generic; +using ln.skyscanner.perfvalue; +using ln.skyscanner.services; +using ln.json.attributes; namespace ln.skyscanner.checks { public enum CheckState { OK, WARN, CRITICAL, FAIL, ERROR } @@ -29,12 +31,9 @@ namespace ln.skyscanner.checks public class SkyCheckState { - [DocumentID] public Guid ID; public readonly String CheckName; - [ByReference] - public readonly Node Node; public PerformanceValue[] PerformanceValues => performanceValues.ToArray(); List performanceValues = new List(); @@ -69,29 +68,6 @@ namespace ln.skyscanner.checks { ID = Guid.NewGuid(); CheckName = skyCheck.Name; - Node = node; - } - - public void WritePerformanceValue(String perfName,double value, double wLower = Double.MinValue, double wUpper = Double.MaxValue, double cLower = Double.MinValue, double cUpper = Double.MaxValue) - { - if (performanceValues == null) - performanceValues = new List(); - - string[] perfPath = new string[] { Node.UniqueIdentity, CheckName, perfName }; - perfName = string.Join("/", perfPath); - - foreach (PerformanceValue performanceValue in performanceValues) - { - if (performanceValue.PerfName.Equals(perfName)) - { - performanceValue.WritePerfValue(value); - return; - } - } - - PerformanceValue newPerformanceValue = new PerformanceValue(perfPath, wLower, wUpper, cLower, cUpper); - performanceValues.Add(newPerformanceValue); - newPerformanceValue.WritePerfValue(value); } public void CheckHistory() @@ -102,6 +78,29 @@ namespace ln.skyscanner.checks } } + public void AddPerformanceValue(PerformanceValue performanceValue) + { + performanceValues.Add(performanceValue); + } + public PerformanceValue GetPerformanceValue(string perfName) + { + foreach (PerformanceValue performanceValue in performanceValues) + { + if (performanceValue.PerfName.Equals(perfName)) + return performanceValue; + } + return null; + } + public bool ContainsPerformanceValue(string perfName) + { + foreach (PerformanceValue performanceValue in performanceValues) + { + if (performanceValue.PerfName.Equals(perfName)) + return true; + } + return false; + } + } diff --git a/checks/SkyChecker.cs b/checks/SkyChecker.cs deleted file mode 100644 index 74201df..0000000 --- a/checks/SkyChecker.cs +++ /dev/null @@ -1,210 +0,0 @@ -// /** -// * File: SkyChecker.cs -// * Author: haraldwolff -// * -// * This file and it's content is copyrighted by the Author and / or copyright holder. -// * Any use wihtout proper permission is illegal and may lead to legal actions. -// * -// * -// **/ -using System; -using System.IO; -using ln.types.btree; -using ln.perfdb.storage; -using ln.skyscanner.entities; -using ln.types.threads; -using System.Threading; -using System.Linq; -using ln.logging; -using ln.snmp; -using ln.http.resources; -using System.Collections.Generic; - -namespace ln.skyscanner.checks -{ - public class SkyChecker - { - public string BasePath { get; } - - [Callable] - public PoolJob[] CurrentJobs => checkPool.CurrentPoolJobs; - [Callable] - public PoolJob[] QueuedJobs => checkPool.QueuedJobs; - - - [Callable] - public string[] PerfNames => perfFiles.Keys.ToArray(); - public bool ContainsPerfFile(String perfName) => perfFiles.ContainsKey(perfName); - - [Callable] - public SkyCheckState[] Issues - { - get - { - SkyCheckState[] checkStates = null; - lock (SkyScanner.Instance.Entities.SkyCheckStates) - { - checkStates = Timing.Meassure("scheduler: checkStates", () => SkyScanner.Instance.Entities.SkyCheckStates.ToArray()); - } - return checkStates.Where((cs) => (!String.Empty.Equals(cs.Node?.Vendor) && ((cs.CheckState != CheckState.OK) || ((cs.UnchangedTime < TimeSpan.FromMinutes(10)) && (cs.History.Length > 1) )))).ToArray(); - } - } - - [Callable] - public SkyCheckState[] CheckStatesByHostID(Guid nodeID) - { - lock (SkyScanner.Instance.Entities.SkyCheckStates) - { - return SkyScanner.Instance.Entities.SkyCheckStates.Query("Node", nodeID).ToArray(); - } - } - - public SNMPEngine SNMPEngine { get; } - - BTree perfFiles = new BTree(); - - Pool checkPool = new Pool(0); - Thread threadScheduler; - - bool stopping; - - List saveQueue = new List(); - - public SkyChecker() - { - BasePath = Path.Combine(SkyScanner.Instance.BasePath, "perfdb"); - - Logging.Log(LogLevel.INFO, "SkyChecker: created"); - - SNMPEngine = new SNMPEngine(); - SNMPEngine.Timeout = 4000; - } - - public void Start() - { - if ((threadScheduler != null) && (threadScheduler.IsAlive)) - throw new NotSupportedException("SkyChecker.scheduler() already running"); - - checkPool.SetPoolSize(256); - - Thread.Sleep(2500); - - threadScheduler = new Thread(scheduler); - threadScheduler.Start(); - } - - public void Stop() - { - stopping = true; - - if (threadScheduler!=null && threadScheduler.IsAlive) - { - threadScheduler.Interrupt(); - } - - checkPool.Abort(); - checkPool.Close(); - - foreach (PerfFile perfFile in perfFiles.Values) - { - if (perfFile.IsOpen) - perfFile.Close(); - } - - perfFiles.Clear(); - } - - public PerfFile GetPerfFile(params string[] perfPath) - { - lock (this) - { - string perfName = string.Join("/", perfPath); - if (!perfFiles.ContainsKey(perfName)) - { - String perfDirectory = Path.Combine(BasePath, Path.Combine(perfPath.Take(perfPath.Length - 1).ToArray())); - if (!Directory.Exists(perfDirectory)) - Directory.CreateDirectory(perfDirectory); - - String perfFileName = Path.Combine(perfDirectory, String.Format("{0}.perf", perfPath[perfPath.Length-1])); - - PerfFile perfFile = new PerfFile(perfFileName); - perfFile.Open(); - - if (perfFile.FirstSection == null) - { - PerfFile.PerfFileSection s1 = new PerfFile.PerfFileSection(perfFile, null, TimeSpan.FromDays(28), 60, AggregationMethod.AVERAGE); - PerfFile.PerfFileSection s2 = new PerfFile.PerfFileSection(perfFile, s1, TimeSpan.FromDays(56), 300, AggregationMethod.AVERAGE); - PerfFile.PerfFileSection s3 = new PerfFile.PerfFileSection(perfFile, s2, TimeSpan.FromDays(84), 900, AggregationMethod.AVERAGE); - PerfFile.PerfFileSection s4 = new PerfFile.PerfFileSection(perfFile, s3, TimeSpan.FromDays(168), 1800, AggregationMethod.AVERAGE); - PerfFile.PerfFileSection s5 = new PerfFile.PerfFileSection(perfFile, s4, TimeSpan.FromDays(750), 3600, AggregationMethod.AVERAGE); - } - perfFile.Close(); - perfFiles.Add(perfName, perfFile); - } - return perfFiles[perfName]; - } - } - - - private void scheduler() - { - long nextMinute = DateTimeOffset.Now.ToUnixTimeMilliseconds(); - - try - { - while (!stopping) - { - List checkJobs = new List(); - - Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): scheduler save CheckStates"); - - lock (saveQueue) - { - foreach (SkyCheckState checkState in saveQueue) - { - SkyScanner.Instance.Entities.SkyCheckStates.Upsert(checkState); - } - saveQueue.Clear(); - } - - Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): scheduler starts"); - - foreach (Node node in SkyScanner.Instance.Entities.NodeCollection) - { - CheckJob checkJob = new CheckJob(node); - checkJobs.Add(checkJob); - } - Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): prepared {0} checks", checkJobs.Count); - Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): scheduled {0} checks", checkPool.Enqueue(checkJobs)); - - while ((nextMinute - DateTimeOffset.Now.ToUnixTimeMilliseconds()) < 0) - nextMinute += 60000; - - Thread.Sleep((int)(nextMinute - DateTimeOffset.Now.ToUnixTimeMilliseconds())); - } - } catch (ThreadInterruptedException) - { - } catch (OperationCanceledException) - { - } - - } - - public void Save(SkyCheckState checkState) - { - lock (saveQueue) - { - saveQueue.Add(checkState); - } - } - - - public ComponentState State - { - get - { - return ComponentState.INITIALIZED; - } - } - } -} diff --git a/checks/Ubiquiti.cs b/checks/Ubiquiti.cs index b7ebfd3..45fc505 100644 --- a/checks/Ubiquiti.cs +++ b/checks/Ubiquiti.cs @@ -17,13 +17,13 @@ namespace ln.skyscanner.checks { } - public override void Check(SkyChecker skyChecker,ref SkyCheckState checkState,Node node) + public override void Check(CheckService checkService,ref SkyCheckState checkState,Node node) { UbiquityCheckState ubiquityCheckState = checkState as UbiquityCheckState; foreach (URI snmpUri in node.FindURIs("snmp")) { - using (SnmpInterface snmp = SnmpInterface.FromURI(snmpUri,skyChecker.SNMPEngine)) + using (SnmpInterface snmp = SnmpInterface.FromURI(snmpUri,checkService.SNMPEngine)) { List mibs = null; try @@ -32,7 +32,9 @@ namespace ln.skyscanner.checks foreach (Sequence s in snmp.snmpWalk("1.3.6.1.2.1.1.9.1.2")) { - Logging.Log(LogLevel.DEBUG, "Ubiquiti: snmpWalk over OIDs: {0}", (s.Items[1] as ObjectIdentifier).AsString); + if (CheckService.DEBUG) + Logging.Log(LogLevel.DEBUG, "Ubiquiti: snmpWalk over OIDs: {0}", (s.Items[1] as ObjectIdentifier).AsString); + mibs.Add((s.Items[1] as ObjectIdentifier).AsString); } @@ -53,10 +55,10 @@ namespace ln.skyscanner.checks foreach (Sequence[] row in ptp) { - ubiquityCheckState.WritePerformanceValue("ptp_rx_capa", (double)((Integer)(row[0].Items[1])).LongValue); - ubiquityCheckState.WritePerformanceValue("ptp_tx_capa", (double)((Integer)(row[1].Items[1])).LongValue); - ubiquityCheckState.WritePerformanceValue("ptp_rx_pwr", (double)((Integer)(row[2].Items[1])).LongValue, -75.0, 0, -80.0, 0); - ubiquityCheckState.WritePerformanceValue("ptp_tx_pwr", (double)((Integer)(row[3].Items[1])).LongValue); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, "ptp_rx_capa", (double)((Integer)(row[0].Items[1])).LongValue); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, "ptp_tx_capa", (double)((Integer)(row[1].Items[1])).LongValue); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, "ptp_rx_pwr", (double)((Integer)(row[2].Items[1])).LongValue, -75.0, 0, -80.0, 0); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, "ptp_tx_pwr", (double)((Integer)(row[3].Items[1])).LongValue); long rxBytes = ((Integer)(row[4].Items[1])).LongValue; long txBytes = ((Integer)(row[5].Items[1])).LongValue; @@ -64,8 +66,8 @@ namespace ln.skyscanner.checks ubiquityCheckState.RXRate.Update(rxBytes * 8); ubiquityCheckState.TXRate.Update(txBytes * 8); - ubiquityCheckState.WritePerformanceValue("ptp_rx_rate", ubiquityCheckState.RXRate.Current); - ubiquityCheckState.WritePerformanceValue("ptp_tx_rate", ubiquityCheckState.TXRate.Current); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, "ptp_rx_rate", ubiquityCheckState.RXRate.Current); + node.WritePerformanceValue(checkService.PerformanceValueService, Name, "ptp_tx_rate", ubiquityCheckState.TXRate.Current); } } } diff --git a/crawl/CrawledHost.cs b/crawl/CrawledHost.cs index 1fce7dd..182fd02 100644 --- a/crawl/CrawledHost.cs +++ b/crawl/CrawledHost.cs @@ -9,19 +9,13 @@ // **/ using System; using System.Collections.Generic; -using ln.types; -using System.Net; using System.Linq; -using ln.snmp.rfc1213; -using ln.types.odb; using ln.types.net; -using ln.types.odb.attributes; namespace ln.skyscanner.crawl { public class CrawledHost { - [DocumentID] public readonly Guid ID = Guid.NewGuid(); public String Name { get; set; } diff --git a/crawl/CrawledSubnet.cs b/crawl/CrawledSubnet.cs index 9bbb71a..9372c84 100644 --- a/crawl/CrawledSubnet.cs +++ b/crawl/CrawledSubnet.cs @@ -8,17 +8,12 @@ // * // **/ using System; -using ln.types; -using ln.types.odb; -using System.Collections.Generic; -using System.Linq; using ln.types.net; -using ln.types.odb.attributes; + namespace ln.skyscanner.crawl { public class CrawledSubnet { - [DocumentID] Guid ID = Guid.NewGuid(); public readonly Network4 Network; diff --git a/crawl/Crawler.cs b/crawl/Crawler.cs index 8f781fa..4c7edb3 100644 --- a/crawl/Crawler.cs +++ b/crawl/Crawler.cs @@ -121,14 +121,14 @@ namespace ln.skyscanner.crawl CrawlerState = ComponentState.STARTED; - if (SkyScanner.Instance.OptionCrawlVendor != null) - { - foreach (Node node in SkyScanner.Instance.Entities.NodeCollection.Query("Vendor", SkyScanner.Instance.OptionCrawlVendor)) - { - CrawledHost crawledHost = FindHostForIP(node.PrimaryIP); - Crawl(crawledHost); - } - } + //if (SkyScanner.Instance.OptionCrawlVendor != null) + //{ + // foreach (Node node in SkyScanner.Instance.Entities.NodeCollection.Query("Vendor", SkyScanner.Instance.OptionCrawlVendor)) + // { + // CrawledHost crawledHost = FindHostForIP(node.PrimaryIP); + // Crawl(crawledHost); + // } + //} } } diff --git a/crawl/service/CrawlService.cs b/crawl/service/CrawlService.cs index 73e253e..3f777af 100644 --- a/crawl/service/CrawlService.cs +++ b/crawl/service/CrawlService.cs @@ -9,7 +9,8 @@ // **/ using System; using System.Collections.Generic; -using ln.skyscanner.crawl.tests; + + namespace ln.skyscanner.crawl.service { public abstract class CrawlService diff --git a/crawl/service/SNMP.cs b/crawl/service/SNMP.cs index ac2bcaa..a4f1960 100644 --- a/crawl/service/SNMP.cs +++ b/crawl/service/SNMP.cs @@ -1,269 +1,269 @@ -// /** -// * File: SNMP.cs -// * Author: haraldwolff -// * -// * This file and it's content is copyrighted by the Author and / or copyright holder. -// * Any use wihtout proper permission is illegal and may lead to legal actions. -// * -// * -// **/ -using System; -using System.Net; -using ln.snmp; -using ln.snmp.endpoint; -using ln.types; -using ln.snmp.types; -using System.Runtime.Remoting.Messaging; -using ln.skyscanner.crawl.service; -using System.Linq; -using ln.types.net; -namespace ln.skyscanner.crawl.tests -{ - public class SNMP : CrawlService - { - public string[] defaultCommunities = new string[0]; +//// /** +//// * File: SNMP.cs +//// * Author: haraldwolff +//// * +//// * This file and it's content is copyrighted by the Author and / or copyright holder. +//// * Any use wihtout proper permission is illegal and may lead to legal actions. +//// * +//// * +//// **/ +//using System; +//using System.Net; +//using ln.snmp; +//using ln.snmp.endpoint; +//using ln.types; +//using ln.snmp.types; +//using System.Runtime.Remoting.Messaging; +//using System.Linq; +//using ln.types.net; - public SNMP(string[] defaultSecrets) - : base("snmp") - { - defaultCommunities = defaultSecrets; - } +//namespace ln.skyscanner.crawl.service +//{ +// public class SNMP : CrawlService +// { +// public string[] defaultCommunities = new string[0]; - public override bool Check(Crawl crawl) - { - if (!crawl.CheckRequiredOption("ping")) - return false; +// public SNMP(string[] defaultSecrets) +// : base("snmp") +// { +// defaultCommunities = defaultSecrets; +// } - if (!TestCurrentHints(crawl.Host)) - { - TestDefaults(crawl.Host); - } +// public override bool Check(Crawl crawl) +// { +// if (!crawl.CheckRequiredOption("ping")) +// return false; - return true; - } +// if (!TestCurrentHints(crawl.Host)) +// { +// TestDefaults(crawl.Host); +// } - public override bool HostProvidesOption(Crawl crawl, params object[] parameters) - { - return (crawl.Host.GetHint("snmp.version",null) != null) && ((crawl.Host.GetHint("snmp.username",null) != null) || (crawl.Host.GetHint("snmp.community",null) != null)); - } +// return true; +// } - public bool TestCurrentHints(CrawledHost crawledHost) - { - SnmpInterface snmpEndpoint = GetSnmpInterface(crawledHost); - if (snmpEndpoint != null) - { - using (snmpEndpoint) - { - try - { - Variable prID = snmpEndpoint.snmpGet("1.3.6.1.2.1.1.2.0"); - } catch (SnmpError) - { - return false; - } catch (TimeoutException) - { - return false; - } - return true; - } - } - return false; - } +// public override bool HostProvidesOption(Crawl crawl, params object[] parameters) +// { +// return (crawl.Host.GetHint("snmp.version",null) != null) && ((crawl.Host.GetHint("snmp.username",null) != null) || (crawl.Host.GetHint("snmp.community",null) != null)); +// } + +// public bool TestCurrentHints(CrawledHost crawledHost) +// { +// SnmpInterface snmpEndpoint = GetSnmpInterface(crawledHost); +// if (snmpEndpoint != null) +// { +// using (snmpEndpoint) +// { +// try +// { +// Variable prID = snmpEndpoint.snmpGet("1.3.6.1.2.1.1.2.0"); +// } catch (SnmpError) +// { +// return false; +// } catch (TimeoutException) +// { +// return false; +// } +// return true; +// } +// } +// return false; +// } - private bool TestSnmpV3(CrawledHost crawledHost) - { - IPv4[] ips = crawledHost.IPAddresses.Concat(new IPv4[] { crawledHost.PrimaryIP }).Distinct().ToArray(); +// private bool TestSnmpV3(CrawledHost crawledHost) +// { +// IPv4[] ips = crawledHost.IPAddresses.Concat(new IPv4[] { crawledHost.PrimaryIP }).Distinct().ToArray(); - foreach (IPv4 ip in ips) - { - using (USMEndpoint v3endpoint = new USMEndpoint(SNMPEngine.DefaultEngine, new IPEndPoint(ip, 161))) - { - try - { - v3endpoint.QueryEngineID(); - } - catch (TimeoutException) - { - } +// foreach (IPv4 ip in ips) +// { +// using (USMEndpoint v3endpoint = new USMEndpoint(SNMPEngine.DefaultEngine, new IPEndPoint(ip, 161))) +// { +// try +// { +// v3endpoint.QueryEngineID(); +// } +// catch (TimeoutException) +// { +// } - if (v3endpoint.RemoteEngineID != null) - { - crawledHost.SetHint("snmp.version", 3); +// if (v3endpoint.RemoteEngineID != null) +// { +// crawledHost.SetHint("snmp.version", 3); - int c = 0; +// int c = 0; - foreach (string community in defaultCommunities) - { - c++; +// foreach (string community in defaultCommunities) +// { +// c++; - v3endpoint.Username = "skytron"; - v3endpoint.AuthMethod = SnmpV3AuthMethod.SHA; - v3endpoint.AuthKeyPhrase = community; +// v3endpoint.Username = "skytron"; +// v3endpoint.AuthMethod = SnmpV3AuthMethod.SHA; +// v3endpoint.AuthKeyPhrase = community; - v3endpoint.LocalizeKeys(); +// v3endpoint.LocalizeKeys(); - try - { - Variable prID = v3endpoint.snmpGet("1.3.6.1.2.1.1.2.0"); +// try +// { +// Variable prID = v3endpoint.snmpGet("1.3.6.1.2.1.1.2.0"); - crawledHost.SetHint("snmp.username", "skytron"); - crawledHost.SetHint("snmp.authkey", community); - crawledHost.SetHint("snmp.address", ip); +// crawledHost.SetHint("snmp.username", "skytron"); +// crawledHost.SetHint("snmp.authkey", community); +// crawledHost.SetHint("snmp.address", ip); - return true; - } - catch (TimeoutException) - { - } - catch (SnmpError) - { - } - } - } - v3endpoint.Close(); - } - } - return false; - } +// return true; +// } +// catch (TimeoutException) +// { +// } +// catch (SnmpError) +// { +// } +// } +// } +// v3endpoint.Close(); +// } +// } +// return false; +// } - private bool TestSnmpV2(CrawledHost crawledHost) - { - IPv4[] ips = crawledHost.IPAddresses.Concat(new IPv4[] { crawledHost.PrimaryIP }).Distinct().ToArray(); +// private bool TestSnmpV2(CrawledHost crawledHost) +// { +// IPv4[] ips = crawledHost.IPAddresses.Concat(new IPv4[] { crawledHost.PrimaryIP }).Distinct().ToArray(); - foreach (IPv4 ip in ips) - { - using (SnmpV2Endpoint v2endpoint = new SnmpV2Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(crawledHost.PrimaryIP, 161))) - { - foreach (String community in defaultCommunities) - { - v2endpoint.CommunityString = community; - try - { - Variable prID = v2endpoint.snmpGet("1.3.6.1.2.1.1.2.0"); +// foreach (IPv4 ip in ips) +// { +// using (SnmpV2Endpoint v2endpoint = new SnmpV2Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(crawledHost.PrimaryIP, 161))) +// { +// foreach (String community in defaultCommunities) +// { +// v2endpoint.CommunityString = community; +// try +// { +// Variable prID = v2endpoint.snmpGet("1.3.6.1.2.1.1.2.0"); - crawledHost.SetHint("snmp.version", 2); - crawledHost.SetHint("snmp.community", community); - crawledHost.SetHint("snmp.address", ip); +// crawledHost.SetHint("snmp.version", 2); +// crawledHost.SetHint("snmp.community", community); +// crawledHost.SetHint("snmp.address", ip); - return true; - } - catch (SnmpError) - { - } - catch (TimeoutException) - { - } - } - v2endpoint.Close(); - } - } - return false; - } +// return true; +// } +// catch (SnmpError) +// { +// } +// catch (TimeoutException) +// { +// } +// } +// v2endpoint.Close(); +// } +// } +// return false; +// } - private bool TestSnmpV1(CrawledHost crawledHost) - { - IPv4[] ips = crawledHost.IPAddresses.Concat(new IPv4[] { crawledHost.PrimaryIP }).Distinct().ToArray(); +// private bool TestSnmpV1(CrawledHost crawledHost) +// { +// IPv4[] ips = crawledHost.IPAddresses.Concat(new IPv4[] { crawledHost.PrimaryIP }).Distinct().ToArray(); - foreach (IPv4 ip in ips) - { - using (SnmpV1Endpoint v1endpoint = new SnmpV1Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(crawledHost.PrimaryIP, 161))) - { - foreach (String community in defaultCommunities) - { - v1endpoint.CommunityString = community; - try - { - Variable prID = v1endpoint.snmpGet("1.3.6.1.2.1.1.2.0"); +// foreach (IPv4 ip in ips) +// { +// using (SnmpV1Endpoint v1endpoint = new SnmpV1Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(crawledHost.PrimaryIP, 161))) +// { +// foreach (String community in defaultCommunities) +// { +// v1endpoint.CommunityString = community; +// try +// { +// Variable prID = v1endpoint.snmpGet("1.3.6.1.2.1.1.2.0"); - crawledHost.SetHint("snmp.version", 1); - crawledHost.SetHint("snmp.community", community); - crawledHost.SetHint("snmp.address", ip); +// crawledHost.SetHint("snmp.version", 1); +// crawledHost.SetHint("snmp.community", community); +// crawledHost.SetHint("snmp.address", ip); - return true; - } - catch (SnmpError) - { - } - catch (TimeoutException) - { - } - } - v1endpoint.Close(); - } - } - return false; - } +// return true; +// } +// catch (SnmpError) +// { +// } +// catch (TimeoutException) +// { +// } +// } +// v1endpoint.Close(); +// } +// } +// return false; +// } - public bool TestDefaults(CrawledHost crawledHost) - { - if (TestSnmpV3(crawledHost) ? true : TestSnmpV2(crawledHost) ? true : TestSnmpV1(crawledHost)) - { - using (SnmpInterface snmp = GetSnmpInterface(crawledHost)) - { - Variable prID = snmp.snmpGet("1.3.6.1.2.1.1.2.0"); - crawledHost.SetHint("snmp.sysObjectID", (prID as ObjectIdentifier).AsString); +// public bool TestDefaults(CrawledHost crawledHost) +// { +// if (TestSnmpV3(crawledHost) ? true : TestSnmpV2(crawledHost) ? true : TestSnmpV1(crawledHost)) +// { +// using (SnmpInterface snmp = GetSnmpInterface(crawledHost)) +// { +// Variable prID = snmp.snmpGet("1.3.6.1.2.1.1.2.0"); +// crawledHost.SetHint("snmp.sysObjectID", (prID as ObjectIdentifier).AsString); - try - { - Sequence[] seqORids = snmp.snmpWalk("1.3.6.1.2.1.1.9.1.2").ToArray(); - string[] ORids = new string[seqORids.Length]; +// try +// { +// Sequence[] seqORids = snmp.snmpWalk("1.3.6.1.2.1.1.9.1.2").ToArray(); +// string[] ORids = new string[seqORids.Length]; - for (int n = 0; n < ORids.Length; n++) - { - ORids[n] = (seqORids[n].Items[1] as ObjectIdentifier).AsString; - } +// for (int n = 0; n < ORids.Length; n++) +// { +// ORids[n] = (seqORids[n].Items[1] as ObjectIdentifier).AsString; +// } - crawledHost.SetHint("snmp.orids", ORids); - } - catch (TimeoutException) - { } - } - return true; - } - else - { - crawledHost.SetHint("snmp.version", null); - crawledHost.SetHint("snmp.username", null); - crawledHost.SetHint("snmp.authkey", null); - crawledHost.SetHint("snmp.community", null); - crawledHost.SetHint("snmp.sysObjectID", null); +// crawledHost.SetHint("snmp.orids", ORids); +// } +// catch (TimeoutException) +// { } +// } +// return true; +// } +// else +// { +// crawledHost.SetHint("snmp.version", null); +// crawledHost.SetHint("snmp.username", null); +// crawledHost.SetHint("snmp.authkey", null); +// crawledHost.SetHint("snmp.community", null); +// crawledHost.SetHint("snmp.sysObjectID", null); - return false; - } - } +// return false; +// } +// } - public static SnmpInterface GetSnmpInterface(CrawledHost crawledHost) - { - int snmpVersion = crawledHost.GetHint("snmp.version", -1); - IPv4 snmpAddress = crawledHost.GetHint("snmp.address", crawledHost.PrimaryIP); +// public static SnmpInterface GetSnmpInterface(CrawledHost crawledHost) +// { +// int snmpVersion = crawledHost.GetHint("snmp.version", -1); +// IPv4 snmpAddress = crawledHost.GetHint("snmp.address", crawledHost.PrimaryIP); - switch (snmpVersion) - { - case 1: - SnmpV1Endpoint v1 = new SnmpV1Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(snmpAddress, 161)); - v1.CommunityString = crawledHost.GetHint("snmp.community"); - return v1; - case 2: - SnmpV2Endpoint v2 = new SnmpV2Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(snmpAddress, 161)); - v2.CommunityString = crawledHost.GetHint("snmp.community"); - return v2; - case 3: - if (crawledHost.GetHint("snmp.username", null) != null) - { - USMEndpoint endpoint = new USMEndpoint(SNMPEngine.DefaultEngine, new System.Net.IPEndPoint(snmpAddress, 161)); - endpoint.AuthMethod = SnmpV3AuthMethod.SHA; - endpoint.Username = crawledHost.GetHint("snmp.username"); - endpoint.AuthKeyPhrase = crawledHost.GetHint("snmp.authkey"); - return endpoint; - } - return null; - default: - return null; - } +// switch (snmpVersion) +// { +// case 1: +// SnmpV1Endpoint v1 = new SnmpV1Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(snmpAddress, 161)); +// v1.CommunityString = crawledHost.GetHint("snmp.community"); +// return v1; +// case 2: +// SnmpV2Endpoint v2 = new SnmpV2Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(snmpAddress, 161)); +// v2.CommunityString = crawledHost.GetHint("snmp.community"); +// return v2; +// case 3: +// if (crawledHost.GetHint("snmp.username", null) != null) +// { +// USMEndpoint endpoint = new USMEndpoint(SNMPEngine.DefaultEngine, new System.Net.IPEndPoint(snmpAddress, 161)); +// endpoint.AuthMethod = SnmpV3AuthMethod.SHA; +// endpoint.Username = crawledHost.GetHint("snmp.username"); +// endpoint.AuthKeyPhrase = crawledHost.GetHint("snmp.authkey"); +// return endpoint; +// } +// return null; +// default: +// return null; +// } - } - } -} +// } +// } +//} diff --git a/crawl/service/SSH.cs b/crawl/service/SSH.cs index 0ba5308..0d343e5 100644 --- a/crawl/service/SSH.cs +++ b/crawl/service/SSH.cs @@ -1,119 +1,119 @@ -// /** -// * File: SSH.cs -// * Author: haraldwolff -// * -// * This file and it's content is copyrighted by the Author and / or copyright holder. -// * Any use wihtout proper permission is illegal and may lead to legal actions. -// * -// * -// **/ -using System; -using Renci.SshNet; -using ln.types; -using Renci.SshNet.Common; -using System.Net.Sockets; -using ln.types.net; -using ln.skyscanner.crawl.service; -using ln.logging; -namespace ln.skyscanner.crawl.tests -{ - public class SSH : CrawlService - { - public SSH() - : base("ssh") - { - } +//// /** +//// * File: SSH.cs +//// * Author: haraldwolff +//// * +//// * This file and it's content is copyrighted by the Author and / or copyright holder. +//// * Any use wihtout proper permission is illegal and may lead to legal actions. +//// * +//// * +//// **/ +//using System; +//using Renci.SshNet; +//using ln.types; +//using Renci.SshNet.Common; +//using System.Net.Sockets; +//using ln.types.net; +//using ln.logging; - public static bool CanConnect(CrawledHost crawledHost) - { - int sshPort = crawledHost.GetHint("ssh.port", -1); - IPv4 sshIP = crawledHost.GetHint("ssh.ip", null); - string sshUser = crawledHost.GetHint("ssh.login", null); - string sshPassword = crawledHost.GetHint("ssh.password", null); +//namespace ln.skyscanner.crawl.service +//{ +// public class SSH : CrawlService +// { +// public SSH() +// : base("ssh") +// { +// } - if ((sshPort == -1) || !CanConnect(crawledHost,sshIP,sshPort,sshUser,sshPassword)) - { - if (!Scan(crawledHost)) - { - crawledHost.SetHint("ssh.port", -1); - crawledHost.SetHint("ssh.ip", null); - crawledHost.SetHint("ssh.login", null); - crawledHost.SetHint("ssh.password", null); - crawledHost.SetHint("ssh.version", null); - return false; - } - } - return true; - } +// public static bool CanConnect(CrawledHost crawledHost) +// { +// int sshPort = crawledHost.GetHint("ssh.port", -1); +// IPv4 sshIP = crawledHost.GetHint("ssh.ip", null); +// string sshUser = crawledHost.GetHint("ssh.login", null); +// string sshPassword = crawledHost.GetHint("ssh.password", null); - private static bool Scan(CrawledHost crawledHost) - { - foreach (IPv4 ip in crawledHost.IPAddresses) - { - foreach (int port in new int[] { 13022, 22 }) - { - if (crawledHost.HasHint(String.Format("tcp.{0}",port)) && crawledHost.GetHint(String.Format("tcp.{0}", port))) - try - { - foreach (Credential credential in SkyScanner.Instance.Crawler.Credentials) - { - Logging.Log(LogLevel.DEBUG, "SSH trying {0}:{1}...", credential.Username, credential.Password.Substring(0, 4)); +// if ((sshPort == -1) || !CanConnect(crawledHost,sshIP,sshPort,sshUser,sshPassword)) +// { +// if (!Scan(crawledHost)) +// { +// crawledHost.SetHint("ssh.port", -1); +// crawledHost.SetHint("ssh.ip", null); +// crawledHost.SetHint("ssh.login", null); +// crawledHost.SetHint("ssh.password", null); +// crawledHost.SetHint("ssh.version", null); +// return false; +// } +// } +// return true; +// } - if (CanConnect(crawledHost, ip, port, credential.Username, credential.Password, true)) - return true; - } - } catch (SocketException) - { - continue; - } - } - } - return false; - } +// private static bool Scan(CrawledHost crawledHost) +// { +// foreach (IPv4 ip in crawledHost.IPAddresses) +// { +// foreach (int port in new int[] { 13022, 22 }) +// { +// if (crawledHost.HasHint(String.Format("tcp.{0}",port)) && crawledHost.GetHint(String.Format("tcp.{0}", port))) +// try +// { +// foreach (Credential credential in SkyScanner.Instance.Crawler.Credentials) +// { +// Logging.Log(LogLevel.DEBUG, "SSH trying {0}:{1}...", credential.Username, credential.Password.Substring(0, 4)); - private static bool CanConnect(CrawledHost crawledHost, IPv4 host, int port, string username, string password,bool throwe = false) - { - using (SshClient client = new SshClient(host.ToString(), port, username, password)) - { - client.ConnectionInfo.Timeout = TimeSpan.FromSeconds(5); - try - { - String authBanner = null; - client.ConnectionInfo.AuthenticationBanner += (object sender, AuthenticationBannerEventArgs e) => authBanner = e.BannerMessage; +// if (CanConnect(crawledHost, ip, port, credential.Username, credential.Password, true)) +// return true; +// } +// } catch (SocketException) +// { +// continue; +// } +// } +// } +// return false; +// } - client.Connect(); +// private static bool CanConnect(CrawledHost crawledHost, IPv4 host, int port, string username, string password,bool throwe = false) +// { +// using (SshClient client = new SshClient(host.ToString(), port, username, password)) +// { +// client.ConnectionInfo.Timeout = TimeSpan.FromSeconds(5); +// try +// { +// String authBanner = null; +// client.ConnectionInfo.AuthenticationBanner += (object sender, AuthenticationBannerEventArgs e) => authBanner = e.BannerMessage; - crawledHost.SetHint("ssh.port", client.ConnectionInfo.Port); - crawledHost.SetHint("ssh.ip", host); - crawledHost.SetHint("ssh.login", client.ConnectionInfo.Username); - crawledHost.SetHint("ssh.password", password); - crawledHost.SetHint("ssh.version", client.ConnectionInfo.ServerVersion); - crawledHost.SetHint("ssh.authbanner", authBanner); +// client.Connect(); - client.Disconnect(); - return true; - } - catch (SshException sshe) - { - Logging.Log(sshe); - } - catch (SocketException) - { - return false; - } - } - return false; - } +// crawledHost.SetHint("ssh.port", client.ConnectionInfo.Port); +// crawledHost.SetHint("ssh.ip", host); +// crawledHost.SetHint("ssh.login", client.ConnectionInfo.Username); +// crawledHost.SetHint("ssh.password", password); +// crawledHost.SetHint("ssh.version", client.ConnectionInfo.ServerVersion); +// crawledHost.SetHint("ssh.authbanner", authBanner); - public override bool Check(Crawl crawl) - { - return CanConnect(crawl.Host); +// client.Disconnect(); +// return true; +// } +// catch (SshException sshe) +// { +// Logging.Log(sshe); +// } +// catch (SocketException) +// { +// return false; +// } +// } +// return false; +// } - } +// public override bool Check(Crawl crawl) +// { +// return CanConnect(crawl.Host); - public override bool HostProvidesOption(Crawl crawl, params object[] parameters) - { - return crawl.Host.GetHint("ssh.ip", null) != null; - } - } -} +// } + +// public override bool HostProvidesOption(Crawl crawl, params object[] parameters) +// { +// return crawl.Host.GetHint("ssh.ip", null) != null; +// } +// } +//} diff --git a/crawl/service/TCP.cs b/crawl/service/TCP.cs index f0a6da3..9eee106 100644 --- a/crawl/service/TCP.cs +++ b/crawl/service/TCP.cs @@ -1,83 +1,83 @@ -// /** -// * File: TCP.cs -// * Author: haraldwolff -// * -// * This file and it's content is copyrighted by the Author and / or copyright holder. -// * Any use wihtout proper permission is illegal and may lead to legal actions. -// * -// * -// **/ -using System; -using System.Net.Sockets; -namespace ln.skyscanner.crawl.service -{ - public class TCP : CrawlService - { - public int[] Ports { get; } +//// /** +//// * File: TCP.cs +//// * Author: haraldwolff +//// * +//// * This file and it's content is copyrighted by the Author and / or copyright holder. +//// * Any use wihtout proper permission is illegal and may lead to legal actions. +//// * +//// * +//// **/ +//using System; +//using System.Net.Sockets; +//namespace ln.skyscanner.crawl.service +//{ +// public class TCP : CrawlService +// { +// public int[] Ports { get; } - public TCP(int[] ports) - :base("tcp") - { - Ports = ports; - } +// public TCP(int[] ports) +// :base("tcp") +// { +// Ports = ports; +// } - public override bool Check(Crawl crawl) - { - if (!crawl.CheckRequiredOption("ping")) - return false; +// public override bool Check(Crawl crawl) +// { +// if (!crawl.CheckRequiredOption("ping")) +// return false; - int n = 0; - foreach (int port in Ports) - { - crawl.UpdateState(this, (double)n / (double)Ports.Length); +// int n = 0; +// foreach (int port in Ports) +// { +// crawl.UpdateState(this, (double)n / (double)Ports.Length); - try - { - using (TcpClient tcpClient = new TcpClient()) - { - IAsyncResult ar = tcpClient.BeginConnect(crawl.Host.PrimaryIP.ToString(), port, null, null); - System.Threading.WaitHandle wh = ar.AsyncWaitHandle; +// try +// { +// using (TcpClient tcpClient = new TcpClient()) +// { +// IAsyncResult ar = tcpClient.BeginConnect(crawl.Host.PrimaryIP.ToString(), port, null, null); +// System.Threading.WaitHandle wh = ar.AsyncWaitHandle; - try - { - if (!ar.AsyncWaitHandle.WaitOne(TimeSpan.FromSeconds(1), false)) - { - tcpClient.Close(); - throw new TimeoutException(); - } +// try +// { +// if (!ar.AsyncWaitHandle.WaitOne(TimeSpan.FromSeconds(1), false)) +// { +// tcpClient.Close(); +// throw new TimeoutException(); +// } - tcpClient.EndConnect(ar); - } - catch (Exception) - { - throw; - } - tcpClient.Close(); +// tcpClient.EndConnect(ar); +// } +// catch (Exception) +// { +// throw; +// } +// tcpClient.Close(); - crawl.Host.SetHint(String.Format("tcp.{0}", port), true); - } - } - catch (Exception) - { - crawl.Host.SetHint(String.Format("tcp.{0}", port), false); - } +// crawl.Host.SetHint(String.Format("tcp.{0}", port), true); +// } +// } +// catch (Exception) +// { +// crawl.Host.SetHint(String.Format("tcp.{0}", port), false); +// } - } - return true; - } +// } +// return true; +// } - public override bool HostProvidesOption(Crawl crawl,params object[] parameters) - { - if (parameters.Length == 0) - return false; +// public override bool HostProvidesOption(Crawl crawl,params object[] parameters) +// { +// if (parameters.Length == 0) +// return false; - foreach (object oport in parameters) - { - int port = (int)oport; - if (!crawl.Host.GetHint(String.Format("tcp.{0}", port),false)) - return false; - } - return true; - } - } -} +// foreach (object oport in parameters) +// { +// int port = (int)oport; +// if (!crawl.Host.GetHint(String.Format("tcp.{0}", port),false)) +// return false; +// } +// return true; +// } +// } +//} diff --git a/crawl/service/Ubiquity.cs b/crawl/service/Ubiquity.cs index 8b2cc6c..d18a7d9 100644 --- a/crawl/service/Ubiquity.cs +++ b/crawl/service/Ubiquity.cs @@ -1,59 +1,58 @@ -// /** -// * File: Ubiquity.cs -// * Author: haraldwolff -// * -// * This file and it's content is copyrighted by the Author and / or copyright holder. -// * Any use wihtout proper permission is illegal and may lead to legal actions. -// * -// * -// **/ -using System; -using System.Linq; -using ln.snmp.endpoint; -using ln.skyscanner.crawl.tests; -using ln.snmp; -using System.Collections.Generic; -using ln.snmp.types; -namespace ln.skyscanner.crawl.service -{ - public class Ubiquity : CrawlService - { - public Ubiquity() - : base("ubiquity") - { - } +//// /** +//// * File: Ubiquity.cs +//// * Author: haraldwolff +//// * +//// * This file and it's content is copyrighted by the Author and / or copyright holder. +//// * Any use wihtout proper permission is illegal and may lead to legal actions. +//// * +//// * +//// **/ +//using System; +//using System.Linq; +//using ln.snmp.endpoint; +//using ln.snmp; +//using System.Collections.Generic; +//using ln.snmp.types; +//namespace ln.skyscanner.crawl.service +//{ +// public class Ubiquity : CrawlService +// { +// public Ubiquity() +// : base("ubiquity") +// { +// } - public override bool Check(Crawl crawl) - { - if (crawl.CheckRequiredOption("snmp")) - { - using (SnmpInterface snmp = SNMP.GetSnmpInterface(crawl.Host)) - { - if (crawl.Host.GetHint("snmp.orids", new string[0]).Contains("1.3.6.1.4.1.41112")) - { - List test = snmp.snmpWalk("1.3.6.1.4.1.41112.1.3.2.1.11"); - //List test2 = snmp.snmpWalk("1.3.6.1.4.1.41112.1.3.3.1.66"); +// public override bool Check(Crawl crawl) +// { +// if (crawl.CheckRequiredOption("snmp")) +// { +// using (SnmpInterface snmp = SNMP.GetSnmpInterface(crawl.Host)) +// { +// if (crawl.Host.GetHint("snmp.orids", new string[0]).Contains("1.3.6.1.4.1.41112")) +// { +// List test = snmp.snmpWalk("1.3.6.1.4.1.41112.1.3.2.1.11"); +// //List test2 = snmp.snmpWalk("1.3.6.1.4.1.41112.1.3.3.1.66"); - if (test.Count > 0) - crawl.Host.SetHint("ubiquity.ptp", true); - else - crawl.Host.SetHint("ubiquity.ptp", false); +// if (test.Count > 0) +// crawl.Host.SetHint("ubiquity.ptp", true); +// else +// crawl.Host.SetHint("ubiquity.ptp", false); - test = snmp.snmpWalk("1.3.6.1.4.1.41112.1.4.7.1.1"); +// test = snmp.snmpWalk("1.3.6.1.4.1.41112.1.4.7.1.1"); - if (test.Count > 0) - crawl.Host.SetHint("ubiquity.ptmp", true); - else - crawl.Host.SetHint("ubiquity.ptmp", false); - } - } - } - return false; - } +// if (test.Count > 0) +// crawl.Host.SetHint("ubiquity.ptmp", true); +// else +// crawl.Host.SetHint("ubiquity.ptmp", false); +// } +// } +// } +// return false; +// } - public override bool HostProvidesOption(Crawl crawl, params object[] parameters) - { - return false; - } - } -} +// public override bool HostProvidesOption(Crawl crawl, params object[] parameters) +// { +// return false; +// } +// } +//} diff --git a/entities/ConfiguredIP.cs b/entities/ConfiguredIP.cs index 94c08b2..8f1804f 100644 --- a/entities/ConfiguredIP.cs +++ b/entities/ConfiguredIP.cs @@ -11,12 +11,11 @@ using System; using ln.types.odb; using ln.types; using ln.types.net; -using ln.types.odb.attributes; + namespace ln.skyscanner.entities { public class ConfiguredIP { - [DocumentID] public readonly Guid ID = Guid.NewGuid(); public IPv4 IP { get; private set; } diff --git a/entities/GlobalNetwork.cs b/entities/GlobalNetwork.cs index ac5d08d..81f675e 100644 --- a/entities/GlobalNetwork.cs +++ b/entities/GlobalNetwork.cs @@ -1,303 +1,301 @@ -using System; -using System.Collections.Generic; -using System.Collections; -using System.Security; -using ln.types; -using System.Linq; -using System.Threading; -using ln.skyscanner.crawl; -using ln.snmp.rfc1213; -using ln.types.odb; -using ln.types.sync; -using ln.logging; -using System.Globalization; -using ln.types.threads; -using ln.types.odb.values; -using ln.types.net; -using System.Runtime.Serialization.Formatters; +//using System; +//using System.Collections.Generic; +//using System.Collections; +//using System.Security; +//using ln.types; +//using System.Linq; +//using System.Threading; +//using ln.snmp.rfc1213; +//using ln.types.odb; +//using ln.logging; +//using System.Globalization; +//using ln.types.threads; +//using ln.types.odb.values; +//using ln.types.net; +//using System.Runtime.Serialization.Formatters; -namespace ln.skyscanner.entities -{ - public class GlobalNetwork : Persistent - { - public HopMap DefaultHopMap { get; private set; } = new HopMap(); - private object _updateLock = new object(); +//namespace ln.skyscanner.entities +//{ +// //public class GlobalNetwork +// //{ +// // public HopMap DefaultHopMap { get; private set; } = new HopMap(); +// // private object _updateLock = new object(); - public GlobalNetwork() - { - } +// // public GlobalNetwork() +// // { +// // } - public void EnsureSubnet(Network4 network) - { - Subnet subnet = SkyScanner.Instance.Entities.SubnetCollection.Where(s => s.Network.Equals(network)).FirstOrDefault(); - if (subnet == null) - { - subnet = new Subnet(network); - SkyScanner.Instance.Entities.SubnetCollection.Upsert(subnet); - } - } +// // public void EnsureSubnet(Network4 network) +// // { +// // Subnet subnet = SkyScanner.Instance.Entities.SubnetCollection.Where(s => s.Network.Equals(network)).FirstOrDefault(); +// // if (subnet == null) +// // { +// // subnet = new Subnet(network); +// // SkyScanner.Instance.Entities.SubnetCollection.Upsert(subnet); +// // } +// // } - public Node GetNode(Guid id) - { - return SkyScanner.Instance.Entities.NodeCollection[id]; - } +// // public Node GetNode(Guid id) +// // { +// // return SkyScanner.Instance.Entities.NodeCollection[id]; +// // } - public Subnet FindSubnetForIP(IPv4 ip) - { - return SkyScanner.Instance.Entities.SubnetCollection.Where((net) => net.Network.Contains(ip)).FirstOrDefault(); - } +// // public Subnet FindSubnetForIP(IPv4 ip) +// // { +// // return SkyScanner.Instance.Entities.SubnetCollection.Where((net) => net.Network.Contains(ip)).FirstOrDefault(); +// // } - public IEnumerable FindHostsInSubnet(Network4 network) - { - ODBValue firstIP = ODBMapper.Default.MapValue(network.First); - ODBValue lastIP = ODBMapper.Default.MapValue(network.Last); +// // public IEnumerable FindHostsInSubnet(Network4 network) +// // { +// // ODBValue firstIP = ODBMapper.Default.MapValue(network.First); +// // ODBValue lastIP = ODBMapper.Default.MapValue(network.Last); - Query nodeByIpQuery = Query.OR( - Query.Equals("Interfaces[].ConfiguredIPs[].Network", network), - Query.IF("PrimaryIP", (ip) => ((firstIP <= ip) && (ip <= lastIP))) - ); - return SkyScanner.Instance.Entities.NodeCollection.Query(nodeByIpQuery); - } +// // Query nodeByIpQuery = Query.OR( +// // Query.Equals("Interfaces[].ConfiguredIPs[].Network", network), +// // Query.IF("PrimaryIP", (ip) => ((firstIP <= ip) && (ip <= lastIP))) +// // ); +// // return SkyScanner.Instance.Entities.NodeCollection.Query(nodeByIpQuery); +// // } - public IEnumerable FindNeighbors(Node node) - { - HashSet networks = new HashSet(); - HashSet nodes = new HashSet(); +// // public IEnumerable FindNeighbors(Node node) +// // { +// // HashSet networks = new HashSet(); +// // HashSet nodes = new HashSet(); - if (node.Interfaces.Count > 0) - { - foreach (Network4 network in node.Networks) - networks.Add(network); - } - Subnet subnet = FindSubnetForIP(node.PrimaryIP); - if (subnet != null) - networks.Add(subnet.Network); +// // if (node.Interfaces.Count > 0) +// // { +// // foreach (Network4 network in node.Networks) +// // networks.Add(network); +// // } +// // Subnet subnet = FindSubnetForIP(node.PrimaryIP); +// // if (subnet != null) +// // networks.Add(subnet.Network); - foreach (Network4 network in networks) - foreach (Node neighbor in FindHostsInSubnet(network)) - nodes.Add(neighbor); +// // foreach (Network4 network in networks) +// // foreach (Node neighbor in FindHostsInSubnet(network)) +// // nodes.Add(neighbor); - return nodes; - } +// // return nodes; +// // } - public Node FindNodeByIP(IEnumerable ips) - { - foreach (IPv4 ip in ips) - { - Node node = FindNodeByIP(ip); - if (node != null) - return node; - } - return null; - } +// // public Node FindNodeByIP(IEnumerable ips) +// // { +// // foreach (IPv4 ip in ips) +// // { +// // Node node = FindNodeByIP(ip); +// // if (node != null) +// // return node; +// // } +// // return null; +// // } - public Node FindNodeByIP(IPv4 ip) - { - Node node = SkyScanner.Instance.Entities.NodeCollection.Query("PrimaryIP", ip).FirstOrDefault(); - if (node == null) - { - Query nodeByIpQuery = Query.Equals("Interfaces[].ConfiguredIPs[].IP", ip); - node = SkyScanner.Instance.Entities.NodeCollection.Query(nodeByIpQuery).FirstOrDefault(); - } - return node; - } +// // public Node FindNodeByIP(IPv4 ip) +// // { +// // Node node = SkyScanner.Instance.Entities.NodeCollection.Query("PrimaryIP", ip).FirstOrDefault(); +// // if (node == null) +// // { +// // Query nodeByIpQuery = Query.Equals("Interfaces[].ConfiguredIPs[].IP", ip); +// // node = SkyScanner.Instance.Entities.NodeCollection.Query(nodeByIpQuery).FirstOrDefault(); +// // } +// // return node; +// // } - [Unsynced] - private Queue updateQueue = new Queue(); +// // [Unsynced] +// // private Queue updateQueue = new Queue(); - public void EnqueueUpdate(CrawledHost crawledHost) - { - lock (updateQueue) - { - if (!updateQueue.Contains(crawledHost)) - updateQueue.Enqueue(crawledHost); - } - } +// // public void EnqueueUpdate(CrawledHost crawledHost) +// // { +// // lock (updateQueue) +// // { +// // if (!updateQueue.Contains(crawledHost)) +// // updateQueue.Enqueue(crawledHost); +// // } +// // } - public void Update() - { - lock (updateQueue) - { - if (updateQueue.Count == 0) - return; - } - while (true) - { - try - { - CrawledHost crawledHost = null; - lock (updateQueue) - { - if (updateQueue.Count > 0) - crawledHost = updateQueue.Dequeue(); - else - break; - } +// // public void Update() +// // { +// // lock (updateQueue) +// // { +// // if (updateQueue.Count == 0) +// // return; +// // } +// // while (true) +// // { +// // try +// // { +// // CrawledHost crawledHost = null; +// // lock (updateQueue) +// // { +// // if (updateQueue.Count > 0) +// // crawledHost = updateQueue.Dequeue(); +// // else +// // break; +// // } - Update(crawledHost); - } catch (Exception e) - { - Logging.Log(LogLevel.ERROR, "GlobalNetwork.Update(): Caught Exception: {0}",e); - Logging.Log(e); - } - } - } +// // Update(crawledHost); +// // } catch (Exception e) +// // { +// // Logging.Log(LogLevel.ERROR, "GlobalNetwork.Update(): Caught Exception: {0}",e); +// // Logging.Log(e); +// // } +// // } +// // } - private void Update(CrawledHost crawledHost) - { - Logging.Log(LogLevel.DEBUG, "Entity: update {0}", crawledHost.PrimaryIP); +// // private void Update(CrawledHost crawledHost) +// // { +// // Logging.Log(LogLevel.DEBUG, "Entity: update {0}", crawledHost.PrimaryIP); - Node node = FindNodeByIP(crawledHost.PrimaryIP); - if (node == null) - node = FindNodeByIP(crawledHost.IPAddresses); +// // Node node = FindNodeByIP(crawledHost.PrimaryIP); +// // if (node == null) +// // node = FindNodeByIP(crawledHost.IPAddresses); - if (node == null) - { - node = new Node(crawledHost.PrimaryIP); - node.Name = crawledHost.Name; - node.PrimaryMac = crawledHost.PrimaryHWAddr; +// // if (node == null) +// // { +// // node = new Node(crawledHost.PrimaryIP); +// // node.Name = crawledHost.Name; +// // node.PrimaryMac = crawledHost.PrimaryHWAddr; - SkyScanner.Instance.Entities.NodeCollection.Insert(node); - } +// // SkyScanner.Instance.Entities.NodeCollection.Insert(node); +// // } - if (!crawledHost.PrimaryIP.Equals(node.PrimaryIP)) - { - Logging.Log(LogLevel.INFO,"GlobalNetwork.Update(): will not update Node with unmatched PrimaryIP {0} != {1}",node.PrimaryIP, crawledHost.PrimaryIP); - return; - } +// // if (!crawledHost.PrimaryIP.Equals(node.PrimaryIP)) +// // { +// // Logging.Log(LogLevel.INFO,"GlobalNetwork.Update(): will not update Node with unmatched PrimaryIP {0} != {1}",node.PrimaryIP, crawledHost.PrimaryIP); +// // return; +// // } - foreach (String si in crawledHost.GetHint("rfc1213.interfaces",new string[0])) - { - String[] fields = si.Split(';'); +// // foreach (String si in crawledHost.GetHint("rfc1213.interfaces",new string[0])) +// // { +// // String[] fields = si.Split(';'); - NetworkInterface networkInterface = node.GetInterface(fields[0]); - if (networkInterface == null) - { - networkInterface = new NetworkInterface(fields[0]); - node.Interfaces.Add(networkInterface); - } +// // NetworkInterface networkInterface = node.GetInterface(fields[0]); +// // if (networkInterface == null) +// // { +// // networkInterface = new NetworkInterface(fields[0]); +// // node.Interfaces.Add(networkInterface); +// // } - networkInterface.HWAddress = fields[1]; +// // networkInterface.HWAddress = fields[1]; - IPv4[][] crawledIPs = fields[2].Split(',').Where((sip) => !String.Empty.Equals(sip)) - .Select((sip) => sip.Split('/').Select(ip => IPv4.Parse(ip)).ToArray()).ToArray(); +// // IPv4[][] crawledIPs = fields[2].Split(',').Where((sip) => !String.Empty.Equals(sip)) +// // .Select((sip) => sip.Split('/').Select(ip => IPv4.Parse(ip)).ToArray()).ToArray(); - ConfiguredIP[] currentIPs = networkInterface.ConfiguredIPs.ToArray(); +// // ConfiguredIP[] currentIPs = networkInterface.ConfiguredIPs.ToArray(); - foreach (IPv4[] crawledIP in crawledIPs) - { - int n; - for (n=0; n < currentIPs.Length; n++) - if (currentIPs[n].IP.Equals(crawledIP[0])) - break; - if (n == currentIPs.Length) - { - ConfiguredIP miss = new ConfiguredIP(crawledIP[0], new Network4(crawledIP[0], crawledIP[1])); - networkInterface.ConfiguredIPs.Add(miss); - } - } +// // foreach (IPv4[] crawledIP in crawledIPs) +// // { +// // int n; +// // for (n=0; n < currentIPs.Length; n++) +// // if (currentIPs[n].IP.Equals(crawledIP[0])) +// // break; +// // if (n == currentIPs.Length) +// // { +// // ConfiguredIP miss = new ConfiguredIP(crawledIP[0], new Network4(crawledIP[0], crawledIP[1])); +// // networkInterface.ConfiguredIPs.Add(miss); +// // } +// // } - foreach (ConfiguredIP iip in currentIPs) - { - IPv4 ip = iip.IP; +// // foreach (ConfiguredIP iip in currentIPs) +// // { +// // IPv4 ip = iip.IP; - if (!crawledIPs.Select((ipl)=>ipl[0]).Contains(ip)) - { - networkInterface.ConfiguredIPs.Remove(iip); - } - } - } +// // if (!crawledIPs.Select((ipl)=>ipl[0]).Contains(ip)) +// // { +// // networkInterface.ConfiguredIPs.Remove(iip); +// // } +// // } +// // } - node.RemoveURI("ssh"); - if (crawledHost.GetHint("ssh.port", -1) != -1) - { - node.AddURI( - new URI( - String.Format("ssh://{0}:{1}@{2}:{3}", - crawledHost.GetHint("ssh.login"), - crawledHost.GetHint("ssh.password"), - crawledHost.GetHint("ssh.ip"), - crawledHost.GetHint("ssh.port") - ) - ) - ); - } +// // node.RemoveURI("ssh"); +// // if (crawledHost.GetHint("ssh.port", -1) != -1) +// // { +// // node.AddURI( +// // new URI( +// // String.Format("ssh://{0}:{1}@{2}:{3}", +// // crawledHost.GetHint("ssh.login"), +// // crawledHost.GetHint("ssh.password"), +// // crawledHost.GetHint("ssh.ip"), +// // crawledHost.GetHint("ssh.port") +// // ) +// // ) +// // ); +// // } - node.RemoveURI("snmp"); - if (crawledHost.GetHint("snmp.version", -1) == 3) - { - node.AddURI(new URI(String.Format("snmp://{0}:{1}:{2}@{3}#{4}", - crawledHost.GetHint("snmp.username",""), - crawledHost.GetHint("snmp.authkey",""), - crawledHost.GetHint("snmp.privkey",""), - crawledHost.GetHint("snmp.address"), - crawledHost.GetHint("snmp.version") - ))); - } - else if (crawledHost.GetHint("snmp.version", -1) > 0) - { - node.AddURI(new URI(String.Format("snmp://{0}@{1}#{2}", - crawledHost.GetHint("snmp.community", ""), - crawledHost.GetHint("snmp.address"), - crawledHost.GetHint("snmp.version") - ))); +// // node.RemoveURI("snmp"); +// // if (crawledHost.GetHint("snmp.version", -1) == 3) +// // { +// // node.AddURI(new URI(String.Format("snmp://{0}:{1}:{2}@{3}#{4}", +// // crawledHost.GetHint("snmp.username",""), +// // crawledHost.GetHint("snmp.authkey",""), +// // crawledHost.GetHint("snmp.privkey",""), +// // crawledHost.GetHint("snmp.address"), +// // crawledHost.GetHint("snmp.version") +// // ))); +// // } +// // else if (crawledHost.GetHint("snmp.version", -1) > 0) +// // { +// // node.AddURI(new URI(String.Format("snmp://{0}@{1}#{2}", +// // crawledHost.GetHint("snmp.community", ""), +// // crawledHost.GetHint("snmp.address"), +// // crawledHost.GetHint("snmp.version") +// // ))); - } +// // } - node.RemoveURI("http"); - if (crawledHost.GetHint("http.port",-1) != -1) - { - node.AddURI(new URI(String.Format("http://{0}:{1}", - crawledHost.GetHint("http.ip", null), - crawledHost.GetHint("http.port")) - )); - } +// // node.RemoveURI("http"); +// // if (crawledHost.GetHint("http.port",-1) != -1) +// // { +// // node.AddURI(new URI(String.Format("http://{0}:{1}", +// // crawledHost.GetHint("http.ip", null), +// // crawledHost.GetHint("http.port")) +// // )); +// // } - string httpIndex = crawledHost.GetHint("http.content", ""); +// // string httpIndex = crawledHost.GetHint("http.content", ""); - if (httpIndex.Contains("RouterOS router configuration page")) - { - node.Vendor = "MicroTik"; - } else if (httpIndex.Contains("mimosa-white-web-logo.png")) - { - node.Vendor = "Mimosa"; - if (httpIndex.Contains("B5c")) - { - node.Product = "B5c"; - node.DeviceType = DeviceType.PTP; - } - } +// // if (httpIndex.Contains("RouterOS router configuration page")) +// // { +// // node.Vendor = "MicroTik"; +// // } else if (httpIndex.Contains("mimosa-white-web-logo.png")) +// // { +// // node.Vendor = "Mimosa"; +// // if (httpIndex.Contains("B5c")) +// // { +// // node.Product = "B5c"; +// // node.DeviceType = DeviceType.PTP; +// // } +// // } - if (crawledHost.GetHint("http.server", "").Equals("Viprinet")) - node.Vendor = "Viprinet"; +// // if (crawledHost.GetHint("http.server", "").Equals("Viprinet")) +// // node.Vendor = "Viprinet"; - if (crawledHost.GetHint("snmp.sysObjectID", "").Equals("1.3.6.1.4.1.318.1.3.27")) - { - node.Vendor = "APC"; - node.DeviceType = DeviceType.UPS; - } else if (crawledHost.GetHint("snmp.orids", new string[0]).Contains("1.3.6.1.4.1.41112")) - { - node.Vendor = "Ubiquity"; - } +// // if (crawledHost.GetHint("snmp.sysObjectID", "").Equals("1.3.6.1.4.1.318.1.3.27")) +// // { +// // node.Vendor = "APC"; +// // node.DeviceType = DeviceType.UPS; +// // } else if (crawledHost.GetHint("snmp.orids", new string[0]).Contains("1.3.6.1.4.1.41112")) +// // { +// // node.Vendor = "Ubiquity"; +// // } - if (crawledHost.GetHint("ubiquity.ptp", false)) - { - node.DeviceType = DeviceType.PTP; - } else if (crawledHost.GetHint("ubiquity.ptmp", false)) - { - node.DeviceType = DeviceType.PTMP; - } +// // if (crawledHost.GetHint("ubiquity.ptp", false)) +// // { +// // node.DeviceType = DeviceType.PTP; +// // } else if (crawledHost.GetHint("ubiquity.ptmp", false)) +// // { +// // node.DeviceType = DeviceType.PTMP; +// // } - foreach (Network4 network in node.Networks) - EnsureSubnet(network); +// // foreach (Network4 network in node.Networks) +// // EnsureSubnet(network); - node.LastUpdate = DateTime.Now; - SkyScanner.Instance.Entities.NodeCollection.Upsert(node); - } - } -} +// // node.LastUpdate = DateTime.Now; +// // SkyScanner.Instance.Entities.NodeCollection.Upsert(node); +// // } +// //} +//} diff --git a/entities/HopMap.cs b/entities/HopMap.cs index 49df802..9c3b18c 100644 --- a/entities/HopMap.cs +++ b/entities/HopMap.cs @@ -15,185 +15,185 @@ using ln.types.btree; using ln.types.net; namespace ln.skyscanner.entities { - public class HopMap : Persistent - { - public Node TopNode { get; private set; } - public GlobalNetwork GlobalNetwork => SkyScanner.Instance.Entities.GlobalNetwork; + //public class HopMap : Persistent + //{ + // public Node TopNode { get; private set; } + // public GlobalNetwork GlobalNetwork => SkyScanner.Instance.Entities.GlobalNetwork; - public int MaxDepth { get; set; } = 16; + // public int MaxDepth { get; set; } = 16; - private BTree hopNodes = new BTree(CompareNodes); + // private BTree hopNodes = new BTree(CompareNodes); - private HashSet networks = new HashSet(); + // private HashSet networks = new HashSet(); - public HopMap() - { - } + // public HopMap() + // { + // } - public HopMap(Node node) - { - Reset(node); - } + // public HopMap(Node node) + // { + // Reset(node); + // } - public void Reset(Node node) - { - hopNodes.Clear(); - TopNode = node; - HopNode hopNode = new HopNode(this, node); - } + // public void Reset(Node node) + // { + // hopNodes.Clear(); + // TopNode = node; + // HopNode hopNode = new HopNode(this, node); + // } - public int GetHopCount(Node node) - { - if (!hopNodes.ContainsKey(node)) - return -1; - return hopNodes[node].HopCount; - } + // public int GetHopCount(Node node) + // { + // if (!hopNodes.ContainsKey(node)) + // return -1; + // return hopNodes[node].HopCount; + // } - public HopNode[] HopNodes - { - get - { - return hopNodes.Values.ToArray(); - } - } + // public HopNode[] HopNodes + // { + // get + // { + // return hopNodes.Values.ToArray(); + // } + // } - public class HopNode : IComparable - { - public HopMap HopMap { get; } - public HopNode Parent { get; private set; } + // public class HopNode : IComparable + // { + // public HopMap HopMap { get; } + // public HopNode Parent { get; private set; } - public Node Node; + // public Node Node; - public HopNode[] Peers => peers.ToArray(); - private HashSet peers = new HashSet(); + // public HopNode[] Peers => peers.ToArray(); + // private HashSet peers = new HashSet(); - private int hopCount = 0; + // private int hopCount = 0; - public HopNode(HopMap hopMap, Node node) - { - HopMap = hopMap; - Node = node; + // public HopNode(HopMap hopMap, Node node) + // { + // HopMap = hopMap; + // Node = node; - HopMap.hopNodes.Add(node, this); + // HopMap.hopNodes.Add(node, this); - FindPeers(); - } - public HopNode(HopNode parent, Node node) - { - HopMap = parent.HopMap; - Node = node; + // FindPeers(); + // } + // public HopNode(HopNode parent, Node node) + // { + // HopMap = parent.HopMap; + // Node = node; - Attach(parent); + // Attach(parent); - FindPeers(); - } + // FindPeers(); + // } - private void FindPeers() - { - foreach (Network4 network in Node.Networks) - { - EnsureNetwork(network); - } - EnsureNetwork(HopMap.GlobalNetwork.FindSubnetForIP(Node.PrimaryIP).Network); - } - private void EnsureNetwork(Network4 network) - { - if (!HopMap.networks.Contains(network)) - { - HopMap.networks.Add(network); - FindPeers(network); - } - } - private void FindPeers(Network4 network) - { - foreach (Node peerNode in HopMap.GlobalNetwork.FindHostsInSubnet(network)) - { - if (!HopMap.hopNodes.ContainsKey(peerNode)) - { - new HopNode(this, peerNode); - } - else - { - HopNode existingHopNode = HopMap.hopNodes[peerNode]; - if (existingHopNode.HopCount > (HopCount + 1)) - { - existingHopNode.Detach(); - existingHopNode.Attach(this); - } - } - } - } + // private void FindPeers() + // { + // foreach (Network4 network in Node.Networks) + // { + // EnsureNetwork(network); + // } + // EnsureNetwork(HopMap.GlobalNetwork.FindSubnetForIP(Node.PrimaryIP).Network); + // } + // private void EnsureNetwork(Network4 network) + // { + // if (!HopMap.networks.Contains(network)) + // { + // HopMap.networks.Add(network); + // FindPeers(network); + // } + // } + // private void FindPeers(Network4 network) + // { + // foreach (Node peerNode in HopMap.GlobalNetwork.FindHostsInSubnet(network)) + // { + // if (!HopMap.hopNodes.ContainsKey(peerNode)) + // { + // new HopNode(this, peerNode); + // } + // else + // { + // HopNode existingHopNode = HopMap.hopNodes[peerNode]; + // if (existingHopNode.HopCount > (HopCount + 1)) + // { + // existingHopNode.Detach(); + // existingHopNode.Attach(this); + // } + // } + // } + // } - public int HopCount - { - get => hopCount; - set - { - hopCount = value; - foreach (HopNode peer in peers) - { - peer.HopCount = hopCount + 1; - } - } - } + // public int HopCount + // { + // get => hopCount; + // set + // { + // hopCount = value; + // foreach (HopNode peer in peers) + // { + // peer.HopCount = hopCount + 1; + // } + // } + // } - public void Attach(HopNode parent) - { - Detach(); + // public void Attach(HopNode parent) + // { + // Detach(); - Parent = parent; - if (!Object.ReferenceEquals(null, Parent)) - { - HopMap.hopNodes.Add(this.Node, this); - Parent.peers.Add(this); + // Parent = parent; + // if (!Object.ReferenceEquals(null, Parent)) + // { + // HopMap.hopNodes.Add(this.Node, this); + // Parent.peers.Add(this); - HopCount = parent.hopCount + 1; - } - else - { - HopCount = 0; - } + // HopCount = parent.hopCount + 1; + // } + // else + // { + // HopCount = 0; + // } - } - public void Detach() - { - if (!Object.ReferenceEquals(null, Parent)) - { - Parent.peers.Remove(this); - Parent = null; - HopMap.hopNodes.Remove(this.Node); - } - } + // } + // public void Detach() + // { + // if (!Object.ReferenceEquals(null, Parent)) + // { + // Parent.peers.Remove(this); + // Parent = null; + // HopMap.hopNodes.Remove(this.Node); + // } + // } - public int CompareTo(object obj) - { - if (obj is HopNode) - { - HopNode you = obj as HopNode; - return Node.ID.CompareTo(you.Node.ID); - } - return this.GetHashCode() - obj.GetHashCode(); - } + // public int CompareTo(object obj) + // { + // if (obj is HopNode) + // { + // HopNode you = obj as HopNode; + // return Node.ID.CompareTo(you.Node.ID); + // } + // return this.GetHashCode() - obj.GetHashCode(); + // } - public IPv4[] Path => _path().ToArray(); - private List _path() - { - if (Parent == null) - return new List(new IPv4[] { Node.PrimaryIP }); - List l = Parent._path(); - l.Add(Node.PrimaryIP); - return l; - } + // public IPv4[] Path => _path().ToArray(); + // private List _path() + // { + // if (Parent == null) + // return new List(new IPv4[] { Node.PrimaryIP }); + // List l = Parent._path(); + // l.Add(Node.PrimaryIP); + // return l; + // } - } + // } - private static int CompareNodes(Node a, Node b) - { - return a.ID.CompareTo(b.ID); - } + // private static int CompareNodes(Node a, Node b) + // { + // return a.ID.CompareTo(b.ID); + // } - } + //} } diff --git a/entities/L2Segment.cs b/entities/L2Segment.cs index 0265651..08efc5e 100644 --- a/entities/L2Segment.cs +++ b/entities/L2Segment.cs @@ -1,12 +1,10 @@ using System; -using ln.types.odb.attributes; using System.Collections.Generic; using ln.types.net; namespace ln.skyscanner.entities { public class L2Segment { - [DocumentID] public Guid ID = Guid.NewGuid(); public String Name; diff --git a/entities/NetworkInterface.cs b/entities/NetworkInterface.cs index 4e35445..a2fd1b7 100644 --- a/entities/NetworkInterface.cs +++ b/entities/NetworkInterface.cs @@ -9,17 +9,11 @@ // **/ using System; using System.Collections.Generic; -using ln.types; -using ln.skyscanner.crawl; -using ln.types.odb; -using ln.logging; using ln.types.net; -using ln.types.odb.attributes; namespace ln.skyscanner.entities { public class NetworkInterface { - [DocumentID] public readonly Guid ID = Guid.NewGuid(); public string Name { get; private set; } = ""; diff --git a/entities/Node.cs b/entities/Node.cs index 6a6a6d8..e767e1d 100644 --- a/entities/Node.cs +++ b/entities/Node.cs @@ -14,12 +14,14 @@ using System.Linq; using ln.types.odb; using ln.types.threads; using ln.types.net; -using ln.types.odb.attributes; +using ln.skyscanner.checks; +using ln.skyscanner.perfvalue; +using ln.json.attributes; +using ln.skyscanner.services; namespace ln.skyscanner.entities { public class Node { - [DocumentID] public Guid ID { get; set; } = Guid.NewGuid(); public IPv4 PrimaryIP { get; set; } @@ -42,10 +44,14 @@ namespace ln.skyscanner.entities public DeviceType DeviceType { get; set; } + [JSONMapping(Private = true)] public List Interfaces { get; private set; } = new List(); + [JSONMapping(Private = true)] public IEnumerable IPAdresses => Interfaces.SelectMany(intf => intf.ConfiguredIPs.Select(nip => nip.IP)); + [JSONMapping(Private = true)] public IEnumerable Networks => Interfaces.SelectMany(intf => intf.ConfiguredIPs.Select(nip => nip.Network)).Distinct(); + [JSONMapping(Private = true)] public IEnumerable Subnets => SkyScanner.Instance.Entities.SubnetCollection.Query(Query.Equals("Network", Networks.Select(net => net.Network))); private string uniqueIdentity; @@ -62,6 +68,14 @@ namespace ln.skyscanner.entities private HashSet uris = new HashSet(); private HashSet checks = new HashSet(); + private Dictionary checkStates = new Dictionary(); + public SkyCheckState[] CheckStates => checkStates.Values.ToArray(); + public string[] CheckStateNames => checkStates.Keys.ToArray(); + + //private Dictionary performanceValues = new Dictionary(); + //public PerformanceValue[] PerformanceValues => performanceValues.Values.ToArray(); + //public string[] PerformanceValueNames => performanceValues.Keys.ToArray(); + private Node() { } @@ -118,6 +132,38 @@ namespace ln.skyscanner.entities return GetInterface(intfName) != null; } + + public bool ContainsCheckState(string name) => checkStates.ContainsKey(name); + public SkyCheckState GetCheckState(string name) => checkStates[name]; + public void AddCheckState(SkyCheckState checkState) + { + checkStates[checkState.CheckName] = checkState; + } + + //public bool ContainsPerformanceValue(String name) => performanceValues.ContainsKey(name); + //public PerformanceValue GetPerformanceValue(string name) => performanceValues[name]; + //public void AddPerformanceValue(PerformanceValue performanceValue) + //{ + // performanceValues[Name] = performanceValue; + //} + + public void WritePerformanceValue(PerformanceValueService performanceValueService, String checkName, String perfName, double value, double wLower = Double.MinValue, double wUpper = Double.MaxValue, double cLower = Double.MinValue, double cUpper = Double.MaxValue) + { + string[] perfPath = new string[] { UniqueIdentity, checkName, perfName }; + perfName = string.Join("/", perfPath); + + SkyCheckState skyCheckState = GetCheckState(checkName); + PerformanceValue performanceValue = skyCheckState.GetPerformanceValue(perfName); + if (performanceValue == null) + { + performanceValue = new PerformanceValue(perfPath, wLower, wUpper, cLower, cUpper); + skyCheckState.AddPerformanceValue(performanceValue); + } + performanceValueService.WriteValue(performanceValue, value); + } + + + public override int GetHashCode() { return ID.GetHashCode(); diff --git a/entities/PointOfPresence.cs b/entities/PointOfPresence.cs index 8747883..ff1d7d9 100644 --- a/entities/PointOfPresence.cs +++ b/entities/PointOfPresence.cs @@ -12,20 +12,15 @@ using ln.types.odb; using System.Collections.Generic; using System.Linq; using ln.types; -using ln.types.odb.attributes; -using ln.types.odb.collections; namespace ln.skyscanner.entities { public class PointOfPresence { - [DocumentID] public Guid ID = Guid.NewGuid(); public String Name { get; set; } public String ForeignName { get; set; } - public RefList Nodes { get; private set; } = new RefList(); - public GeoLocation GeoLocation { get; set; } public DateTime Created { get; private set; } = DateTime.Now; @@ -33,7 +28,7 @@ namespace ln.skyscanner.entities { get { - return SkyScanner.Instance.Entities.L2SegmentCollection.Query("PoPs[]", this); + throw new NotImplementedException(); } } @@ -41,10 +36,6 @@ namespace ln.skyscanner.entities { } - public void AddNode(Node node) => Nodes.Add(node); - public void RemoveNode(Node node) => Nodes.Remove(node); - - } } diff --git a/entities/Subnet.cs b/entities/Subnet.cs index 5c57f88..bdf3331 100644 --- a/entities/Subnet.cs +++ b/entities/Subnet.cs @@ -13,14 +13,10 @@ using ln.types.odb; using System.Collections.Generic; using System.Linq; using ln.types.net; -using ln.types.odb.attributes; namespace ln.skyscanner.entities { - public class Subnet : Persistent + public class Subnet { - public SkyEntities SkyEntities => SkyScanner.Instance.Entities; - - [DocumentID] public Guid ID { get; private set; } = Guid.NewGuid(); public readonly Network4 Network; diff --git a/http/CheckerApi.cs b/http/CheckerApi.cs deleted file mode 100644 index dfebf7c..0000000 --- a/http/CheckerApi.cs +++ /dev/null @@ -1,130 +0,0 @@ -// /** -// * File: CheckerApi.cs -// * Author: haraldwolff -// * -// * This file and it's content is copyrighted by the Author and / or copyright holder. -// * Any use wihtout proper permission is illegal and may lead to legal actions. -// * -// * -// **/ -using System; -using System.Collections.Generic; -using ln.http; -using ln.http.resources; -using ln.types.threads; -using ln.perfdb.storage; -using System.Net; -using ln.skyscanner.checks; -using ln.types.odb; -using System.Linq; -using ln.json.mapping; -namespace ln.skyscanner.http -{ - public class CheckerApi : JsonCallResource - { - public CheckerApi(Resource container) - : base(container, "checker") - { - new Checks(this); - } - - [Callable] - public PoolJob[] CurrentPoolJobs => SkyScanner.Instance.Checker.CurrentJobs; - [Callable] - public PoolJob[] QueuedPoolJobs => SkyScanner.Instance.Checker.QueuedJobs; - [Callable] - public SkyCheckState[] Issues - { - get - { - Query issueQuery = Query.EqualsNot("CheckState", CheckState.OK); - return SkyScanner.Instance.Entities.SkyCheckStates.Query(issueQuery).ToArray(); - } - } - - - - - - class Checks : BaseResource - { - public Checks(CheckerApi container) - :base(container,"checks") - { } - - public override bool HandlesDispatching => true; - - public override bool Contains(string name) - { - return base.Contains(name) || SkyScanner.Instance.Checker.ContainsPerfFile(name); - } - public override Resource GetResource(string name) - { - if (base.Contains(name)) - return base.GetResource(name); - - if (SkyScanner.Instance.Checker.ContainsPerfFile(name)) - { - return new Check(this, name); - } - - throw new KeyNotFoundException(); - } - public override HttpResponse GetResponse(HttpRequest httpRequest, Queue pathStack) - { - if (pathStack.Count == 0) - return GetResponse(httpRequest); - - string perfName = WebUtility.UrlDecode(String.Join("/", pathStack)); - - Resource perfResource = GetResource(perfName); - return perfResource.GetResponse(httpRequest); - } - - public override HttpResponse GetResponse(HttpRequest httpRequest) - { - HttpResponse response = new HttpResponse(httpRequest); - response.SetHeader("content-type", "application/json"); - response.ContentWriter.Write(JSONMapper.DefaultMapper.ToJson(SkyScanner.Instance.Checker.PerfNames)); - return response; - } - - class Check : Resource - { - public Check(Checks container,String perfName) - :base(container,perfName) - {} - - public override void AddResource(Resource resource) => throw new NotImplementedException(); - public override bool Contains(string name) => throw new NotImplementedException(); - public override IEnumerable GetResources() => throw new NotImplementedException(); - public override void RemoveResource(Resource resource) => throw new NotImplementedException(); - - public override HttpResponse GetResponse(HttpRequest httpRequest) - { - PerfFile perfFile = SkyScanner.Instance.Checker.GetPerfFile(Name); - PerfValue[] perfValues = null; - - int interval = 3600; - - if (httpRequest.Query.ContainsKey("interval")) - interval = int.Parse(httpRequest.Query["interval"]); - - lock (perfFile) - { - perfFile.EnsureOpen(); - perfValues = perfFile.QueryTime(interval, 0); - perfFile.Close(); - } - - HttpResponse response = new HttpResponse(httpRequest); - response.SetHeader("content-type", "application/json"); - response.ContentWriter.Write(JSONMapper.DefaultMapper.ToJson(perfValues).ToString()); - - return response; - } - } - - } - } -} diff --git a/http/CrawlerApi.cs b/http/CrawlerApi.cs index 8cf1db7..0b21d7a 100644 --- a/http/CrawlerApi.cs +++ b/http/CrawlerApi.cs @@ -1,145 +1,144 @@ -using System; -using ln.http.resources; -using ln.skyscanner.entities; -using System.Linq; -using ln.skyscanner.crawl; -using ln.types; -using System.Net; -using ln.types.threads; -using ln.types.net; +//using System; +//using ln.http.resources; +//using ln.skyscanner.entities; +//using System.Linq; +//using ln.types; +//using System.Net; +//using ln.types.threads; +//using ln.types.net; -namespace ln.skyscanner.http -{ - public class CrawlerApi : JsonCallResource - { - public SkyScanner SkyScanner { get; } - public Crawler Crawler => SkyScanner.Crawler; +//namespace ln.skyscanner.http +//{ +// public class CrawlerApi : JsonCallResource +// { +// public SkyScanner SkyScanner { get; } +// public Crawler Crawler => SkyScanner.Crawler; - public CrawlerApi(Resource container, SkyScanner skyScanner) - : base(container, "crawler") - { - SkyScanner = skyScanner; - } +// public CrawlerApi(Resource container, SkyScanner skyScanner) +// : base(container, "crawler") +// { +// SkyScanner = skyScanner; +// } - [Callable] - public CrawledSubnet[] GetSubnets() - { - return SkyScanner.Instance.Entities.CrawledSubnets.ToArray(); - } +// [Callable] +// public CrawledSubnet[] GetSubnets() +// { +// return SkyScanner.Instance.Entities.CrawledSubnets.ToArray(); +// } - [Callable] - public CrawledHost[] GetHosts() - { - CrawledHost[] result = SkyScanner.Instance.Entities.CrawledHosts.ToArray(); - return result; - } +// [Callable] +// public CrawledHost[] GetHosts() +// { +// CrawledHost[] result = SkyScanner.Instance.Entities.CrawledHosts.ToArray(); +// return result; +// } - [Callable] - public CrawledHost GetHostByIP(string _ip) - { - return SkyScanner.Crawler.FindHostForIP(IPv4.Parse(_ip)); - } +// [Callable] +// public CrawledHost GetHostByIP(string _ip) +// { +// return SkyScanner.Crawler.FindHostForIP(IPv4.Parse(_ip)); +// } - [Callable] - public CrawledHost AddHost(string _ip, string name) - { - IPv4 ip = IPv4.Parse(_ip); - CrawledHost host = SkyScanner.Crawler.FindHostForIP(ip); - host.Name = name; - SkyScanner.Instance.Entities.CrawledHosts.Upsert(host); - return host; - } +// [Callable] +// public CrawledHost AddHost(string _ip, string name) +// { +// IPv4 ip = IPv4.Parse(_ip); +// CrawledHost host = SkyScanner.Crawler.FindHostForIP(ip); +// host.Name = name; +// SkyScanner.Instance.Entities.CrawledHosts.Upsert(host); +// return host; +// } - [Callable] - public void TriggerRecrawl() - { - foreach (CrawledHost host in SkyScanner.Instance.Entities.CrawledHosts.ToArray()) - { - host.NextCheck = DateTime.Now; - SkyScanner.Instance.Entities.CrawledHosts.Upsert(host); - } - } +// [Callable] +// public void TriggerRecrawl() +// { +// foreach (CrawledHost host in SkyScanner.Instance.Entities.CrawledHosts.ToArray()) +// { +// host.NextCheck = DateTime.Now; +// SkyScanner.Instance.Entities.CrawledHosts.Upsert(host); +// } +// } - [Callable] - public void Crawl(string _id) - { - Guid hostID = Guid.Parse(_id); - SkyScanner.Crawler.Crawl(hostID); - } +// [Callable] +// public void Crawl(string _id) +// { +// Guid hostID = Guid.Parse(_id); +// SkyScanner.Crawler.Crawl(hostID); +// } - [Callable] - public PoolJob[] GetCurrentPoolJobs() - { - return SkyScanner.Crawler.CurrentJobs; - } +// [Callable] +// public PoolJob[] GetCurrentPoolJobs() +// { +// return SkyScanner.Crawler.CurrentJobs; +// } - [Callable] - public PoolJob[] GetQueuedJobs() - { - return SkyScanner.Crawler.QueuedJobs; - } +// [Callable] +// public PoolJob[] GetQueuedJobs() +// { +// return SkyScanner.Crawler.QueuedJobs; +// } - [Callable] - public bool GetCrawlSubnets() - { - return SkyScanner.Crawler.CrawlSubnets; - } - [Callable] - public void SetCrawlSubnets(bool enable) - { - SkyScanner.Crawler.CrawlSubnets = enable; - } +// [Callable] +// public bool GetCrawlSubnets() +// { +// return SkyScanner.Crawler.CrawlSubnets; +// } +// [Callable] +// public void SetCrawlSubnets(bool enable) +// { +// SkyScanner.Crawler.CrawlSubnets = enable; +// } - [Callable] - public bool GetCrawlHosts() - { - return SkyScanner.Crawler.CrawlHosts; - } - [Callable] - public void SetCrawlHosts(bool enable) - { - SkyScanner.Crawler.CrawlHosts = enable; - } +// [Callable] +// public bool GetCrawlHosts() +// { +// return SkyScanner.Crawler.CrawlHosts; +// } +// [Callable] +// public void SetCrawlHosts(bool enable) +// { +// SkyScanner.Crawler.CrawlHosts = enable; +// } - [Callable] - public void ToggleCrawlHosts() - { - SkyScanner.Crawler.CrawlHosts = !SkyScanner.Crawler.CrawlHosts; - } - [Callable] - public void ToggleCrawlSubnets() - { - SkyScanner.Crawler.CrawlSubnets = !SkyScanner.Crawler.CrawlSubnets; - } +// [Callable] +// public void ToggleCrawlHosts() +// { +// SkyScanner.Crawler.CrawlHosts = !SkyScanner.Crawler.CrawlHosts; +// } +// [Callable] +// public void ToggleCrawlSubnets() +// { +// SkyScanner.Crawler.CrawlSubnets = !SkyScanner.Crawler.CrawlSubnets; +// } - [Callable] - public Network4[] GetBlockedNetworks() - { - return SkyScanner.Instance.Entities.BlockedNetworks.ToArray(); - } - [Callable] - public void AddBlockedNetwork(string _network) - { - Network4 blocked = Network4.Parse(_network); - if (!SkyScanner.Instance.Entities.BlockedNetworks.Contains(blocked)) - { - SkyScanner.Instance.Entities.BlockedNetworks.Insert(blocked); - } - } - [Callable] - public void RemoveBlockedNetwork(string _network) - { - Network4 blocked = Network4.Parse(_network); - foreach (Network4 blockedNetwork in SkyScanner.Instance.Entities.BlockedNetworks) - { - if (blockedNetwork.Equals(blocked)) - { - SkyScanner.Instance.Entities.BlockedNetworks.Delete(blockedNetwork); - return; - } - } - } +// [Callable] +// public Network4[] GetBlockedNetworks() +// { +// return SkyScanner.Instance.Entities.BlockedNetworks.ToArray(); +// } +// [Callable] +// public void AddBlockedNetwork(string _network) +// { +// Network4 blocked = Network4.Parse(_network); +// if (!SkyScanner.Instance.Entities.BlockedNetworks.Contains(blocked)) +// { +// SkyScanner.Instance.Entities.BlockedNetworks.Insert(blocked); +// } +// } +// [Callable] +// public void RemoveBlockedNetwork(string _network) +// { +// Network4 blocked = Network4.Parse(_network); +// foreach (Network4 blockedNetwork in SkyScanner.Instance.Entities.BlockedNetworks) +// { +// if (blockedNetwork.Equals(blocked)) +// { +// SkyScanner.Instance.Entities.BlockedNetworks.Delete(blockedNetwork); +// return; +// } +// } +// } - } -} +// } +//} diff --git a/http/NetworkApi.cs b/http/NetworkApi.cs index d1111bb..cfb3518 100644 --- a/http/NetworkApi.cs +++ b/http/NetworkApi.cs @@ -17,42 +17,40 @@ namespace ln.skyscanner.http { public class NetworkApi : JsonCallResource { - public GlobalNetwork GlobalNetwork => SkyScanner.Instance.Entities.GlobalNetwork; - public NetworkApi(SkyScannerHttpApi container) :base(container,"network") { } - [Callable] - public Node[] GetNeighbors(string pid) - { - Guid nodeID = Guid.Parse(pid); - return GlobalNetwork.FindNeighbors(GlobalNetwork.GetNode(nodeID)).ToArray(); - } + //[Callable] + //public Node[] GetNeighbors(string pid) + //{ + // Guid nodeID = Guid.Parse(pid); + // return GlobalNetwork.FindNeighbors(GlobalNetwork.GetNode(nodeID)).ToArray(); + //} - [Callable] - public Node[] GetHostsInSubnet(string _network) - { - Network4 network = Network4.Parse(_network); - return GlobalNetwork.FindHostsInSubnet(network).ToArray(); - } + //[Callable] + //public Node[] GetHostsInSubnet(string _network) + //{ + // Network4 network = Network4.Parse(_network); + // return GlobalNetwork.FindHostsInSubnet(network).ToArray(); + //} - [Callable] - public HopMap.HopNode[] GetHopTable() - { - HopMap hopMap = new HopMap(GlobalNetwork.FindNodeByIP(IPv4.Parse("10.10.10.2"))); - return hopMap.HopNodes; - } + //[Callable] + //public HopMap.HopNode[] GetHopTable() + //{ + // HopMap hopMap = new HopMap(GlobalNetwork.FindNodeByIP(IPv4.Parse("10.10.10.2"))); + // return hopMap.HopNodes; + //} - [Callable] - public HopMap.HopNode[] GetHopTable(string pid) - { - Guid persistenceID = Guid.Parse(pid); - HopMap hopMap = new HopMap(GlobalNetwork.GetNode(persistenceID)); - return hopMap.HopNodes; - } + //[Callable] + //public HopMap.HopNode[] GetHopTable(string pid) + //{ + // Guid persistenceID = Guid.Parse(pid); + // HopMap hopMap = new HopMap(GlobalNetwork.GetNode(persistenceID)); + // return hopMap.HopNodes; + //} } } diff --git a/http/SkyScannerHttpApi.cs b/http/SkyScannerHttpApi.cs index 57a59dc..b1a458b 100644 --- a/http/SkyScannerHttpApi.cs +++ b/http/SkyScannerHttpApi.cs @@ -13,11 +13,6 @@ namespace ln.skyscanner.http :base(skyScannerHttpApplication.RootResource,"api") { SkyScanner = skyScannerHttpApplication.SkyScanner; - - new SkyScannerHttpManagement(this,SkyScanner); - new CrawlerApi(this, SkyScanner); - new NetworkApi(this); - new CheckerApi(this); } diff --git a/http/SkyScannerHttpApplication.cs b/http/SkyScannerHttpApplication.cs index 2a3819f..08b4619 100644 --- a/http/SkyScannerHttpApplication.cs +++ b/http/SkyScannerHttpApplication.cs @@ -1,14 +1,11 @@ using System; -using ln.http; using ln.http.resources; using System.IO; using ln.skyscanner.entities; using ln.skyscanner.checks; -using ln.skyscanner.crawl; using ln.types.net; using System.Linq; -using System.Collections.Generic; -using ln.types.odb; +using ln.skyscanner.perfvalue; namespace ln.skyscanner.http { public class SkyScannerHttpApplication : ResourceApplication diff --git a/ln.skyscanner.csproj b/ln.skyscanner.csproj index cc286a7..4edb9c0 100644 --- a/ln.skyscanner.csproj +++ b/ln.skyscanner.csproj @@ -62,8 +62,6 @@ - - @@ -73,43 +71,29 @@ - - - - - - - - - - - - - - - - + + @@ -241,6 +225,21 @@ PreserveNewest + + + + + + + + + + + + + + + @@ -262,6 +261,7 @@ + diff --git a/checks/PerformanceValue.cs b/perfvalue/PerformanceValue.cs similarity index 80% rename from checks/PerformanceValue.cs rename to perfvalue/PerformanceValue.cs index fa30102..9c59cd7 100644 --- a/checks/PerformanceValue.cs +++ b/perfvalue/PerformanceValue.cs @@ -9,7 +9,9 @@ // **/ using System; using ln.perfdb.storage; -namespace ln.skyscanner.checks +using ln.skyscanner.checks; + +namespace ln.skyscanner.perfvalue { public class PerformanceValue { @@ -39,19 +41,6 @@ namespace ln.skyscanner.checks CritUpper = cUpper; } - public void WritePerfValue(double value) - { - LastValue = value; - - PerfFile perfFile = SkyScanner.Instance.Checker.GetPerfFile(PerfPath); - lock (perfFile) - { - perfFile.EnsureOpen(); - perfFile.Write(DateTimeOffset.Now.ToUnixTimeSeconds(), value); - perfFile.Close(); - } - } - public CheckState CheckState { get diff --git a/services/CheckService.cs b/services/CheckService.cs index b286f29..41e5082 100644 --- a/services/CheckService.cs +++ b/services/CheckService.cs @@ -8,48 +8,57 @@ using ln.skyscanner.checks; using System.Collections.Generic; using ln.types.threads; using System.Linq; -using ln.types.odb; +using ln.snmp; +using ln.types.odb.ng; namespace ln.skyscanner.services { public class CheckService : ApplicationServiceBase { + public static bool DEBUG = false; + Pool checkPool; EntityService entityService; + PerformanceValueService performanceValueService; + + public EntityService EntityService => entityService; + public PerformanceValueService PerformanceValueService => performanceValueService; + public SNMPEngine SNMPEngine { get; private set; } + + SkyScanner.Service coreService; + CheckServiceRPC RPC; public CheckService() :base("Check Service") { + DependOnService(); DependOnService(); + DependOnService(); } public override void ServiceMain(IApplicationInterface applicationInterface) { + coreService = Dependency(); entityService = Dependency(); + performanceValueService = Dependency(); Dictionary checkJobs = new Dictionary(); HashSet currentNodes = new HashSet(); - checkPool = new Pool(64); + checkPool = new Pool(96); + + SNMPEngine = new SNMPEngine(); + SNMPEngine.Timeout = 2000; long nextMinute = DateTimeOffset.Now.ToUnixTimeMilliseconds(); + RPC = new CheckServiceRPC(this); + coreService.RPCContainer.Add("CheckService",RPC); + Ready(); while (!StopRequested) { - /* - Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): scheduler save CheckStates"); - - lock (saveQueue) - { - foreach (SkyCheckState checkState in saveQueue) - { - SkyScanner.Instance.Entities.SkyCheckStates.Upsert(checkState); - } - saveQueue.Clear(); - } - */ Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): scheduler starts"); currentNodes.Clear(); @@ -58,7 +67,7 @@ namespace ln.skyscanner.services currentNodes.Add(node); if (!checkJobs.ContainsKey(node)) { - CheckJob checkJob = new CheckJob(node); + CheckJob checkJob = new CheckJob(this,node); checkJobs.Add(node, checkJob); } } @@ -70,17 +79,20 @@ namespace ln.skyscanner.services if (currentNodes.Count > 0) { foreach (Node node in currentNodes) + { checkJobs.Remove(node); + } } currentNodes.Clear(); foreach (CheckJob checkJob in checkJobs.Values) { - + checkJob.Prepare(); } - Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): prepared {0} checks", checkJobs.Count); - Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): scheduled {0} checks", checkPool.Enqueue(checkJobs.Values)); + Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): currently knows {0} checks", checkJobs.Count); + Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): currently {0} checks alive", checkPool.NumQueuedJobs); + Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): scheduled {0} checks", checkPool.Enqueue(checkJobs.Values)); while (true) @@ -91,6 +103,7 @@ namespace ln.skyscanner.services try { Thread.Sleep((int)(nextMinute - DateTimeOffset.Now.ToUnixTimeMilliseconds())); + break; } catch (ThreadInterruptedException) { Logging.Log(LogLevel.INFO, "CheckService: scheduler was interrupted"); @@ -100,13 +113,16 @@ namespace ln.skyscanner.services } } + coreService.RPCContainer.Remove("CheckService"); entityService = null; + performanceValueService = null; + coreService = null; } public SkyCheckState[] GetCheckStates(Node node) { Query stateQuery = Query.Equals("Node", node.ID); - SkyCheckState[] skyCheckStates = entityService.SkyCheckStates.Query(stateQuery).ToArray(); + SkyCheckState[] skyCheckStates = entityService.SkyCheckStates.SelectQuery(stateQuery).ToArray(); Dictionary checkStates = new Dictionary(); foreach (SkyCheckState checkState in skyCheckStates) @@ -135,6 +151,24 @@ namespace ln.skyscanner.services this.checkService = checkService; } + public Node[] GetIssueList() + { + List nodes = new List(); + + foreach (Node node in checkService.EntityService.NodeCollection) + { + foreach (SkyCheckState checkState in node.CheckStates) + { + if (!String.Empty.Equals(node.Vendor) && ((checkState.CheckState != CheckState.OK) || ((checkState.UnchangedTime < TimeSpan.FromMinutes(10)) && (checkState.History.Length > 1)))){ + nodes.Add(node); + break; + } + } + } + + return nodes.ToArray(); + } + } } diff --git a/services/EntityService.cs b/services/EntityService.cs index 8a9f061..d01f710 100644 --- a/services/EntityService.cs +++ b/services/EntityService.cs @@ -3,43 +3,37 @@ using ln.application; using ln.application.service; using System.Threading; using System.IO; -using ln.types.odb; -using ln.types.odb.mapped; +using ln.types.odb.ng; using ln.skyscanner.entities; -using ln.skyscanner.crawl; -using ln.types.net; -using ln.skyscanner.checks; using ln.logging; +using System.Collections.Generic; +using ln.types.odb.ng.storage; namespace ln.skyscanner.services { public class EntityService : ApplicationServiceBase { public string BasePath { get; private set; } - public ODB ODB { get; private set; } - - public ODBCollection NodeCollection { get; private set; } - public ODBCollection SubnetCollection { get; private set; } - - public ODBCollection PointOfPresenceCollection { get; private set; } - public ODBCollection L2SegmentCollection { get; private set; } - - public ODBCollection CrawledHosts { get; private set; } - public ODBCollection CrawledSubnets { get; private set; } - - public ODBCollection BlockedNetworks { get; private set; } - - public ODBCollection SkyCheckStates { get; private set; } + public SkyScanner.Service CoreService { get; private set; } + public RPC RPCInstance { get; private set; } + public IStorageContainer StorageContainer { get; private set; } + public IStorageContainer StorageSession { get; private set; } + public Mapper SessionMapper { get; private set; } + Queue upsertQueue = new Queue(); public EntityService() :base("Entity Service") { + DependOnService(); + RPCInstance = new RPC(this); } public override void ServiceMain(IApplicationInterface applicationInterface) { + CoreService = Dependency(); + BasePath = Path.Combine( CurrentApplicationInterface.Arguments["base-path"].Value, "entities" @@ -47,48 +41,82 @@ namespace ln.skyscanner.services Logging.Log(LogLevel.INFO, "Entity Service: Initializing"); - ODB = new ODB(BasePath); + StorageContainer = new FSStorageContainer(System.IO.Path.Combine(BasePath, "storage")); + StorageSession = new Session(StorageContainer); + SessionMapper = new Mapper(StorageSession); - NodeCollection = ODB.GetCollection(); - NodeCollection.EnableStrongCache(true); - NodeCollection.EnsureIndex("uniqueIdentity", true); - NodeCollection.EnsureIndeces( - "PrimaryIP", - "Interfaces[].ConfiguredIPs[].IP", - "Interfaces[].ConfiguredIPs[].Network" - ); - - SubnetCollection = ODB.GetCollection(); - SubnetCollection.EnsureIndeces("Network"); - - PointOfPresenceCollection = ODB.GetCollection(); - PointOfPresenceCollection.EnsureIndeces("ForeignName"); - - L2SegmentCollection = ODB.GetCollection(); - L2SegmentCollection.EnsureIndeces("Network", "PoPs"); - - CrawledHosts = ODB.GetCollection(); - CrawledHosts.EnsureIndeces("PrimaryIP", "IPAddresses[]"); - - CrawledSubnets = ODB.GetCollection(); - BlockedNetworks = ODB.GetCollection("blockedNetworks"); - - SkyCheckStates = ODB.GetCollection(); - SkyCheckStates.EnableStrongCache(true); - SkyCheckStates.EnsureIndeces("Node"); - SkyCheckStates.EnsureUniqueness("Node", "CheckName"); + StorageContainer.Open(); + StorageSession.Open(); + CoreService.RPCContainer.Add("entities",RPCInstance); Ready(); - lock (Thread.CurrentThread) + while (!StopRequested) { - Monitor.Wait(Thread.CurrentThread); + //object o; + //while ( + // (o = DequeueUpsert()) != null + // ) + //{ + // ODB.GetCollection(o.GetType()).Upsert(o); + //} + lock (upsertQueue) + { + if (upsertQueue.Count == 0) + Monitor.Wait(upsertQueue,2500); + } } + CoreService.RPCContainer.Remove(RPCInstance); + CoreService = null; + + SessionMapper = null; + StorageSession.Dispose(); + StorageContainer.Dispose(); + BasePath = null; } + public void EnqueueUpsert(T o) where T: class + { + //ObjectCollection col = ODBMapper.GetCollection(); + //lock (upsertQueue) + //{ + // upsertQueue.Enqueue(o); + // Monitor.Pulse(upsertQueue); + //} + } + + public object DequeueUpsert() + { + lock (upsertQueue) + { + if (upsertQueue.Count == 0) + return null; + return upsertQueue.Dequeue(); + } + } + + + public class RPC + { + EntityService EntityService { get; } + + public RPC(EntityService entityService) + { + EntityService = entityService; + } + + public Node GetNode(Guid nodeID) + { + return EntityService.SessionMapper.Load(nodeID); + } + + + } + + } } diff --git a/services/PerformanceValueService.cs b/services/PerformanceValueService.cs new file mode 100644 index 0000000..fc2b5a2 --- /dev/null +++ b/services/PerformanceValueService.cs @@ -0,0 +1,117 @@ +using System; +using ln.application; +using ln.application.service; +using System.Threading; +using ln.skyscanner.perfvalue; +using ln.perfdb.storage; +using System.Collections.Generic; +using System.Linq; +using System.IO; +namespace ln.skyscanner.services +{ + public class PerformanceValueService : ApplicationServiceBase + { + public String BasePath { get; private set; } + + private Dictionary perfFiles = new Dictionary(); + public PerfFile[] PerfFiles => perfFiles.Values.ToArray(); + + SkyScanner.Service coreService; + public RPC RPCInstance { get; private set; } + + public PerformanceValueService() + :base("PerformanceValueService") + { + DependOnService(); + RPCInstance = new RPC(this); + } + + public override void ServiceMain(IApplicationInterface applicationInterface) + { + coreService = Dependency(); + + BasePath = Path.Combine( + CurrentApplicationInterface.Arguments["base-path"].Value, + "perfdb" + ); + + coreService.RPCContainer.Add("perfValues", RPCInstance); + + Ready(); + + while (!StopRequested) + { + lock (this) + { + Monitor.Wait(this); + } + } + + coreService.RPCContainer.Remove(RPCInstance); + coreService = null; + } + + public PerfFile GetPerfFile(params string[] perfPath) + { + lock (this) + { + string perfName = string.Join("/", perfPath); + if (!perfFiles.ContainsKey(perfName)) + { + String perfDirectory = Path.Combine(BasePath, Path.Combine(perfPath.Take(perfPath.Length - 1).ToArray())); + if (!Directory.Exists(perfDirectory)) + Directory.CreateDirectory(perfDirectory); + + String perfFileName = Path.Combine(perfDirectory, String.Format("{0}.perf", perfPath[perfPath.Length - 1])); + + PerfFile perfFile = new PerfFile(perfFileName); + perfFile.Open(); + + if (perfFile.FirstSection == null) + { + PerfFile.PerfFileSection s1 = new PerfFile.PerfFileSection(perfFile, null, TimeSpan.FromDays(28), 60, AggregationMethod.AVERAGE); + PerfFile.PerfFileSection s2 = new PerfFile.PerfFileSection(perfFile, s1, TimeSpan.FromDays(56), 300, AggregationMethod.AVERAGE); + PerfFile.PerfFileSection s3 = new PerfFile.PerfFileSection(perfFile, s2, TimeSpan.FromDays(84), 900, AggregationMethod.AVERAGE); + PerfFile.PerfFileSection s4 = new PerfFile.PerfFileSection(perfFile, s3, TimeSpan.FromDays(168), 1800, AggregationMethod.AVERAGE); + PerfFile.PerfFileSection s5 = new PerfFile.PerfFileSection(perfFile, s4, TimeSpan.FromDays(750), 3600, AggregationMethod.AVERAGE); + } + perfFile.Close(); + perfFiles.Add(perfName, perfFile); + } + return perfFiles[perfName]; + } + } + + public void WriteValue(PerformanceValue performanceValue,double value) + { + performanceValue.LastValue = value; + + PerfFile perfFile = GetPerfFile(performanceValue.PerfPath); + lock (perfFile) + { + perfFile.EnsureOpen(); + perfFile.Write(DateTimeOffset.Now.ToUnixTimeSeconds(), value); + perfFile.Close(); + } + } + + public class RPC + { + PerformanceValueService PerformanceValueService { get; } + public RPC(PerformanceValueService performanceValueService) + { + PerformanceValueService = performanceValueService; + } + + public PerfValue[] GetPerfData(string[] perfPath,int timeWindow = 3600) + { + PerfFile perfFile = PerformanceValueService.GetPerfFile(perfPath); + perfFile.EnsureOpen(); + PerfValue[] perfData = perfFile.QueryTime(timeWindow, 0); + return perfData; + } + } + + + } +} diff --git a/templates/static/checks/issues2.html b/templates/static/checks/issues2.html index 6b30cfa..d4e1e9f 100644 --- a/templates/static/checks/issues2.html +++ b/templates/static/checks/issues2.html @@ -23,10 +23,15 @@ diff --git a/templates/static/checks/node.html b/templates/static/checks/node.html index 0090b80..7a6086d 100644 --- a/templates/static/checks/node.html +++ b/templates/static/checks/node.html @@ -1,6 +1,23 @@ <%frame "frame.html"%> -

Knoten ()

+

Knoten ()

+
+
+ Current State:
+ Sender:
+
+
+ +
HTTP:
+ + +
SSH:
+ + +
TELNET:
+ +
+
Graphen Intervall: @@ -14,6 +31,8 @@
+ Checks + @@ -33,17 +52,86 @@ + diff --git a/templates/static/system/index.html b/templates/static/system/index.html index f8ca5d2..307b577 100644 --- a/templates/static/system/index.html +++ b/templates/static/system/index.html @@ -47,23 +47,23 @@ // $("#cbCrawlHosts").on("click", function(){ skyapi().call("api/crawler","ToggleCrawlHosts"); } ); // $("#cbCrawlSubnets").on("click", function(){ skyapi().call("api/crawler","ToggleCrawlSubnets"); } ); - function showServices(json){ - if (json.result) + function showServices(result,error){ + if (result) { $("#ServiceTable > tbody").empty(); - json.result.forEach(element => { + result.forEach(element => { var row = $("") .attr("id",$.escapeSelector(element)) .append($("").text(element)); $("#ServiceTable > tbody").append(row); - LN().rpc("ServiceContainer","GetServiceDefinition",[element,],function(json){ - console.log(JSON.stringify(json)); + LN().rpc("ServiceContainer","GetServiceDefinition",[element,],function(result,error){ + console.log(JSON.stringify(result)); row.append( - $("").attr("bool",json.result.IsLoaded), - $("").attr("bool",json.result.IsAlive), - $("").attr("bool",json.result.IsReady) + $("").attr("bool",result.IsLoaded), + $("").attr("bool",result.IsAlive), + $("").attr("bool",result.IsReady) ); });