From 9905238db567a201dbe3af8dab6b4794dcd66545 Mon Sep 17 00:00:00 2001 From: Harald Wolff Date: Thu, 4 Apr 2019 19:34:19 +0200 Subject: [PATCH] WIP --- Program.cs | 2 +- SkyScanner.cs | 16 ++++++++- checks/Hostalive.cs | 1 + checks/SkyCheck.cs | 1 + checks/SkyChecker.cs | 4 +++ checks/Ubiquity.cs | 54 +++++++++++++++++++++++++++++ crawl/Crawler.cs | 41 +++++++++++++++------- crawl/service/ICMP.cs | 2 ++ crawl/service/RFC1213.cs | 2 ++ crawl/service/SNMP.cs | 1 + entities/GlobalNetwork.cs | 21 +++++++++++ http/CrawlerApi.cs | 10 ++++++ ln.skyscanner.csproj | 1 + templates/static/checks/index.html | 7 ++++ templates/static/crawlerhosts.html | 1 + templates/static/network/index.html | 2 +- templates/static/skyapi.js | 15 ++++++++ 17 files changed, 165 insertions(+), 16 deletions(-) create mode 100644 checks/Ubiquity.cs diff --git a/Program.cs b/Program.cs index 5f0ec87..3b70788 100644 --- a/Program.cs +++ b/Program.cs @@ -31,7 +31,7 @@ namespace ln.skyscanner fileLogger.MaxLogLevel = LogLevel.INFO; Logger.Default.Backends.Add(fileLogger); - //Logger.ConsoleLogger.MaxLogLevel = LogLevel.DEBUGDETAIL; + Logger.ConsoleLogger.MaxLogLevel = LogLevel.INFO; Initialize(); diff --git a/SkyScanner.cs b/SkyScanner.cs index 01f6ce7..c7ac213 100644 --- a/SkyScanner.cs +++ b/SkyScanner.cs @@ -11,6 +11,7 @@ using sharp.logging; using Newtonsoft.Json; using Newtonsoft.Json.Converters; using ln.skyscanner.checks; +using System.Collections.Generic; namespace ln.skyscanner { [JsonConverter(typeof(StringEnumConverter))] @@ -40,7 +41,7 @@ namespace ln.skyscanner Logging.Log(LogLevel.INFO, "SkyScanner: Constructor"); - BasePath = Path.GetFullPath("."); + BasePath = Path.GetFullPath("/var/cache/ln.skyscanner"); Logging.Log(LogLevel.INFO, "SkyScanner: BasePath={0}", BasePath); @@ -48,6 +49,19 @@ namespace ln.skyscanner Arguments = args; + Queue qArguments = new Queue(Arguments); + while (qArguments.Count > 0) + { + string arg = qArguments.Dequeue(); + switch (arg) + { + case "-p": + BasePath = Path.GetFullPath(qArguments.Dequeue()); + break; + } + } + + Entities = new SkyEntities(this); Checker = new SkyChecker(); } diff --git a/checks/Hostalive.cs b/checks/Hostalive.cs index 9665817..360aac7 100644 --- a/checks/Hostalive.cs +++ b/checks/Hostalive.cs @@ -26,6 +26,7 @@ namespace ln.skyscanner.checks Ping = new Ping(); } + public override bool IsCritical => true; public override bool IsValid(Node node) => node.PrimaryIP != null; public override void Check(SkyChecker skyChecker, Node node) diff --git a/checks/SkyCheck.cs b/checks/SkyCheck.cs index b5d3f2f..fcae637 100644 --- a/checks/SkyCheck.cs +++ b/checks/SkyCheck.cs @@ -38,6 +38,7 @@ namespace ln.skyscanner.checks static SkyCheck() { AddSkyCheck(new Hostalive()); + AddSkyCheck(new Ubiquity()); } } } diff --git a/checks/SkyChecker.cs b/checks/SkyChecker.cs index 39e0ae6..807bdec 100644 --- a/checks/SkyChecker.cs +++ b/checks/SkyChecker.cs @@ -16,6 +16,7 @@ using ln.types.threads; using System.Threading; using System.Linq; using ln.logging; +using ln.snmp; namespace ln.skyscanner.checks { public class SkyChecker @@ -28,6 +29,8 @@ namespace ln.skyscanner.checks public string[] PerfNames => perfFiles.Keys.ToArray(); public bool ContainsPerfFile(String perfName) => perfFiles.ContainsKey(perfName); + public SNMPEngine SNMPEngine { get; } + BTree perfFiles = new BTree(); Pool checkPool = new Pool(0); @@ -38,6 +41,7 @@ namespace ln.skyscanner.checks public SkyChecker() { BasePath = Path.Combine(SkyScanner.Instance.BasePath, "perfdb"); + SNMPEngine = new SNMPEngine(); } public void Start() diff --git a/checks/Ubiquity.cs b/checks/Ubiquity.cs new file mode 100644 index 0000000..7d3255b --- /dev/null +++ b/checks/Ubiquity.cs @@ -0,0 +1,54 @@ +using System; +using ln.skyscanner.entities; +using ln.types; +using ln.snmp; +using System.Collections.Generic; +using ln.snmp.types; +using ln.logging; + +namespace ln.skyscanner.checks +{ + public class Ubiquity : SkyCheck + { + public Ubiquity() + :base("ubiquity") + { + } + + + public override void Check(SkyChecker skyChecker, Node node) + { + foreach (URI snmpUri in node.FindURIs("snmp")) + { + using (SnmpInterface snmp = SnmpInterface.FromURI(snmpUri,skyChecker.SNMPEngine)) + { + Sequence[][] ptp = snmp.snmpWalk(new string[] { + "1.3.6.1.4.1.41112.1.3.2.1.5", + "1.3.6.1.4.1.41112.1.3.2.1.6", + "1.3.6.1.4.1.41112.1.3.2.1.11", + "1.3.6.1.4.1.41112.1.3.2.1.14", + "1.3.6.1.4.1.41112.1.3.3.1.64", + "1.3.6.1.4.1.41112.1.3.3.1.66" + }); + + foreach (Sequence[] row in ptp) + { + skyChecker.WritePerfValue(this, "ptp_rx_capa", node, (double)((Integer)(row[0].Items[1])).LongValue); + skyChecker.WritePerfValue(this, "ptp_tx_capa", node, (double)((Integer)(row[1].Items[1])).LongValue); + skyChecker.WritePerfValue(this, "ptp_rx_pwr", node, (double)((Integer)(row[2].Items[1])).LongValue); + skyChecker.WritePerfValue(this, "ptp_tx_pwr", node, (double)((Integer)(row[3].Items[1])).LongValue); + + skyChecker.WritePerfValue(this, "ptp_rx_rate", node, (double)((Integer)(row[4].Items[1])).LongValue); // ToDo: multiply 8 / delta T + skyChecker.WritePerfValue(this, "ptp_tx_rate", node, (double)((Integer)(row[5].Items[1])).LongValue); // ToDo: multiply 8 / delta T + } + + } + } + } + + public override bool IsValid(Node node) + { + return (node.Vendor != null) && node.Vendor.Equals("Ubiquity"); + } + } +} diff --git a/crawl/Crawler.cs b/crawl/Crawler.cs index 5db253c..67d0c2a 100644 --- a/crawl/Crawler.cs +++ b/crawl/Crawler.cs @@ -33,6 +33,7 @@ namespace ln.skyscanner.crawl CrawlService.RegisterService(new RFC1213()); CrawlService.RegisterService(new HTTP()); CrawlService.RegisterService(new SSH()); + CrawlService.RegisterService(new Ubiquity()); } public SkyScanner SkyScanner { get; } @@ -85,6 +86,9 @@ namespace ln.skyscanner.crawl subnets = odbDatabase.GetCollection(); blockedNetworks = odbDatabase.GetCollection("blockedNetworks"); + hosts.EnsureIndex("PrimaryIP"); + hosts.EnsureIndex("IPAddresses[]"); + CrawlerState = ComponentState.INITIALIZED; } catch (Exception) @@ -190,27 +194,38 @@ namespace ln.skyscanner.crawl public CrawledHost FindHostForIP(IPv4 ip) { - CrawledHost crawledHost = CrawledHosts.Where(host => host.HasIP(ip)).FirstOrDefault(); - if (crawledHost == null) + lock (this) { - crawledHost = new CrawledHost(); - crawledHost.PrimaryIP = ip; - crawledHost.Name = ip.ToString(); + Query nodeByIpQuery = Query.OR( + Query.Equals("IPAddresses[]", ip), + Query.Equals("PrimaryIP", ip) + ); + CrawledHost crawledHost = hosts.Select(nodeByIpQuery).FirstOrDefault(); + if (crawledHost == null) + { + crawledHost = new CrawledHost(); + crawledHost.PrimaryIP = ip; + crawledHost.Name = ip.ToString(); - CrawledHosts.Insert(crawledHost); + CrawledHosts.Insert(crawledHost); + } + return crawledHost; } - return crawledHost; } public CrawledSubnet FindSubnet(Network4 network) { - CrawledSubnet sn = CrawledSubnets.Select("Network", network).FirstOrDefault(); - if (sn == null) + lock (this) { - Logging.Log(LogLevel.INFO, "Crawler adds new subnet: {0}",network); - sn = new CrawledSubnet(network); - CrawledSubnets.Insert(sn); + Query subnetQuery = Query.Equals("Network", network); + CrawledSubnet sn = CrawledSubnets.Select(subnetQuery).FirstOrDefault(); + if (sn == null) + { + Logging.Log(LogLevel.INFO, "Crawler adds new subnet: {0}", network); + sn = new CrawledSubnet(network); + CrawledSubnets.Insert(sn); + } + return sn; } - return sn; } diff --git a/crawl/service/ICMP.cs b/crawl/service/ICMP.cs index 5720514..3a264ab 100644 --- a/crawl/service/ICMP.cs +++ b/crawl/service/ICMP.cs @@ -50,6 +50,8 @@ namespace ln.skyscanner.crawl.tests crawl.Host.SetHint("ping.out_of_ten", nSuccess); crawl.Host.LastSeen = DateTime.Now; + if (crawl.Host.FirstSeen < new DateTime(1970, 1, 2)) + crawl.Host.FirstSeen = DateTime.Now; } else { diff --git a/crawl/service/RFC1213.cs b/crawl/service/RFC1213.cs index b09f9ee..a8f205e 100644 --- a/crawl/service/RFC1213.cs +++ b/crawl/service/RFC1213.cs @@ -15,6 +15,7 @@ using ln.types; using ln.skyscanner.crawl.service; using System.Configuration; using System.Net.Sockets; +using ln.types.net; namespace ln.skyscanner.crawl.tests { public class RFC1213 : CrawlService @@ -48,6 +49,7 @@ namespace ln.skyscanner.crawl.tests { crawl.Host.IPAddresses = interfaces.SelectMany(intf => intf.IPAddresses).ToArray(); crawl.Host.HWAddresses = interfaces.Select(intf => intf.HWAddr).ToArray(); + crawl.Host.Networks = interfaces.SelectMany(intf => intf.IPAddresses).Zip(interfaces.SelectMany(intf => intf.Netmasks), (ip, nm) => new Network4(ip.AsUInt, nm.AsUInt)).ToArray(); crawl.Host.SetHint("rfc1213.interfaces", interfaces.Select(intf => string.Format("{0};{1};{2}", intf.Name, diff --git a/crawl/service/SNMP.cs b/crawl/service/SNMP.cs index 4e33f72..ac2bcaa 100644 --- a/crawl/service/SNMP.cs +++ b/crawl/service/SNMP.cs @@ -38,6 +38,7 @@ namespace ln.skyscanner.crawl.tests { TestDefaults(crawl.Host); } + return true; } diff --git a/entities/GlobalNetwork.cs b/entities/GlobalNetwork.cs index 2bc42e5..ace566e 100644 --- a/entities/GlobalNetwork.cs +++ b/entities/GlobalNetwork.cs @@ -238,6 +238,27 @@ namespace ln.skyscanner.entities ); } + node.RemoveURI("snmp"); + if (crawledHost.GetHint("snmp.version", -1) == 3) + { + node.AddURI(new URI(String.Format("snmp://{0}:{1}:{2}@{3}#{4}", + crawledHost.GetHint("snmp.username",""), + crawledHost.GetHint("snmp.authkey",""), + crawledHost.GetHint("snmp.privkey",""), + crawledHost.GetHint("snmp.address"), + crawledHost.GetHint("snmp.version") + ))); + } + else if (crawledHost.GetHint("snmp.version", -1) > 0) + { + node.AddURI(new URI(String.Format("snmp://{0}@{1}#{2}", + crawledHost.GetHint("snmp.community", ""), + crawledHost.GetHint("snmp.address"), + crawledHost.GetHint("snmp.version") + ))); + + } + node.RemoveURI("http"); if (crawledHost.GetHint("http.port",-1) != -1) diff --git a/http/CrawlerApi.cs b/http/CrawlerApi.cs index 8e545e5..c6bd9eb 100644 --- a/http/CrawlerApi.cs +++ b/http/CrawlerApi.cs @@ -50,6 +50,16 @@ namespace ln.skyscanner.http return host; } + [Callable] + public void TriggerRecrawl() + { + foreach (CrawledHost host in Crawler.CrawledHosts.ToArray()) + { + host.NextCheck = DateTime.Now; + Crawler.CrawledHosts.Upsert(host); + } + } + [Callable] public void Crawl(string _id) { diff --git a/ln.skyscanner.csproj b/ln.skyscanner.csproj index d21690b..6431c72 100644 --- a/ln.skyscanner.csproj +++ b/ln.skyscanner.csproj @@ -82,6 +82,7 @@ + diff --git a/templates/static/checks/index.html b/templates/static/checks/index.html index 69f1a84..1a83bbe 100644 --- a/templates/static/checks/index.html +++ b/templates/static/checks/index.html @@ -27,6 +27,13 @@ }, options: { scales: { + yAxes: [ + { + ticks: { + callback: ScaleSI + } + } + ], xAxes: [{ type: 'time', time: { diff --git a/templates/static/crawlerhosts.html b/templates/static/crawlerhosts.html index f28b82a..ee61dba 100644 --- a/templates/static/crawlerhosts.html +++ b/templates/static/crawlerhosts.html @@ -5,6 +5,7 @@
+
Bezeichnung: diff --git a/templates/static/network/index.html b/templates/static/network/index.html index 85a175c..1056725 100644 --- a/templates/static/network/index.html +++ b/templates/static/network/index.html @@ -60,7 +60,7 @@ $("#dLocation").prop("disabled", false).val(node.Location.Latitude + " / " + node.Location.Longitude); $("#dURIs").empty(); - $.each( node.URIs, function(){ $("#dURIs").append($("" + this.Scheme + "://" + this.Host + ":" + this.Port + "
")); } ); + $.each( node.URIs, function(){ $("#dURIs").append($("" + this.Scheme + "://" + this.Host + ":" + this.Port + (this.Fragment ? "#" + this.Fragment : "") + "
")); } ); $("#dInterfaces").empty(); $.each( node.Interfaces, function(){ diff --git a/templates/static/skyapi.js b/templates/static/skyapi.js index cbec0d4..ccdf266 100644 --- a/templates/static/skyapi.js +++ b/templates/static/skyapi.js @@ -176,3 +176,18 @@ function skyapi() { return __skyapi; } + + + +function ScaleSI(value) +{ + if (value > 1000000000) + return ((value / 1000000000) | 0) + "G"; + if (value > 1000000) + return ((value / 1000000) | 0) + "M"; + if (value > 1000) + return ((value / 1000) | 0) + "k"; + return value; +} + +