From 07062a2c3abc267f8fb37b8f5ba8811ac156e98c Mon Sep 17 00:00:00 2001 From: Harald Wolff Date: Fri, 28 Jun 2019 10:01:30 +0200 Subject: [PATCH] WIP --- Program.cs | 1 + SkyEntities.cs | 5 + SkyScanner.cs | 16 +- checks/SkyCheck.cs | 2 +- checks/SkyChecker.cs | 2 +- checks/{Ubiquity.cs => Ubiquiti.cs} | 8 +- entities/L2Segment.cs | 23 +++ entities/Node.cs | 1 + entities/PointOfPresence.cs | 9 + http/SkyScannerHttpManagement.cs | 2 +- import/skytron/SkytronImport.cs | 200 +++++++++++++++++++++++ ln.skyscanner.csproj | 30 +++- packages.config | 5 +- templates/static/checks/issues.html | 70 +++++--- templates/static/css/style.css | 49 ++++-- templates/static/frame.html | 2 +- templates/static/network/index.html | 3 +- templates/static/network/l2segments.html | 107 ++++++++++++ templates/static/topnav.html | 2 +- 19 files changed, 490 insertions(+), 47 deletions(-) rename checks/{Ubiquity.cs => Ubiquiti.cs} (96%) create mode 100644 entities/L2Segment.cs create mode 100644 import/skytron/SkytronImport.cs create mode 100644 templates/static/network/l2segments.html diff --git a/Program.cs b/Program.cs index df08ca2..624b813 100644 --- a/Program.cs +++ b/Program.cs @@ -19,6 +19,7 @@ using ln.types.odb.index; using ln.types.odb.values; using ln.skyscanner.checks; using System.Linq; +using ln.skyscanner.import.skytron; namespace ln.skyscanner { diff --git a/SkyEntities.cs b/SkyEntities.cs index 04b09e5..a4e4f1e 100644 --- a/SkyEntities.cs +++ b/SkyEntities.cs @@ -34,7 +34,9 @@ namespace ln.skyscanner public ODBCollection NodeCollection { get; private set; } public ODBCollection SubnetCollection { get; private set; } + public ODBCollection PointOfPresenceCollection { get; private set; } + public ODBCollection L2SegmentCollection { get; private set; } public ODBCollection CrawledHosts { get; private set; } public ODBCollection CrawledSubnets { get; private set; } @@ -75,6 +77,9 @@ namespace ln.skyscanner PointOfPresenceCollection = ODB.GetCollection(); PointOfPresenceCollection.EnsureIndeces("ForeignName"); + L2SegmentCollection = ODB.GetCollection(); + L2SegmentCollection.EnsureIndeces("Network","PoPs"); + CrawledHosts = ODB.GetCollection(); CrawledHosts.EnsureIndeces("PrimaryIP","IPAddresses[]"); diff --git a/SkyScanner.cs b/SkyScanner.cs index dda841b..17c9067 100644 --- a/SkyScanner.cs +++ b/SkyScanner.cs @@ -7,11 +7,12 @@ using System.IO; using ln.skyscanner.crawl; using ln.types.threads; using System.Threading; -using sharp.logging; +using ln.logging; using Newtonsoft.Json; using Newtonsoft.Json.Converters; using ln.skyscanner.checks; using System.Collections.Generic; +using ln.skyscanner.import.skytron; namespace ln.skyscanner { [JsonConverter(typeof(StringEnumConverter))] @@ -49,6 +50,9 @@ namespace ln.skyscanner Arguments = args; + String skytronImport = null; + + Queue qArguments = new Queue(Arguments); while (qArguments.Count > 0) { @@ -58,11 +62,21 @@ namespace ln.skyscanner case "-p": BasePath = Path.GetFullPath(qArguments.Dequeue()); break; + case "--import-skytron": + skytronImport = qArguments.Dequeue(); + break; } } Entities = new SkyEntities(this); Checker = new SkyChecker(); + + if (skytronImport != null) + { + SkytronImport si = new SkytronImport(skytronImport); + si.Import(); + } + } private SkyScanner() { diff --git a/checks/SkyCheck.cs b/checks/SkyCheck.cs index 0721275..f14eb31 100644 --- a/checks/SkyCheck.cs +++ b/checks/SkyCheck.cs @@ -41,7 +41,7 @@ namespace ln.skyscanner.checks static SkyCheck() { AddSkyCheck(new Hostalive()); - AddSkyCheck(new Ubiquity()); + AddSkyCheck(new Ubiquiti()); AddSkyCheck(new APC()); } } diff --git a/checks/SkyChecker.cs b/checks/SkyChecker.cs index b5781c8..f2ae2de 100644 --- a/checks/SkyChecker.cs +++ b/checks/SkyChecker.cs @@ -78,7 +78,7 @@ namespace ln.skyscanner.checks if ((threadScheduler != null) && (threadScheduler.IsAlive)) throw new NotSupportedException("SkyChecker.scheduler() already running"); - checkPool.SetPoolSize(256); + checkPool.SetPoolSize(128); Thread.Sleep(2500); diff --git a/checks/Ubiquity.cs b/checks/Ubiquiti.cs similarity index 96% rename from checks/Ubiquity.cs rename to checks/Ubiquiti.cs index 1fbd34e..141b1a1 100644 --- a/checks/Ubiquity.cs +++ b/checks/Ubiquiti.cs @@ -8,10 +8,10 @@ using ln.logging; namespace ln.skyscanner.checks { - public class Ubiquity : SkyCheck + public class Ubiquiti : SkyCheck { - public Ubiquity() - :base("ubiquity") + public Ubiquiti() + :base("ubiquiti") { } @@ -57,7 +57,7 @@ namespace ln.skyscanner.checks public override bool IsValid(Node node) { - return (node.Vendor != null) && node.Vendor.Equals("Ubiquity"); + return (node.Vendor != null) && node.Vendor.Equals("Ubiquiti"); } public override SkyCheckState PrepareCheckState(SkyChecker skyChecker, Node node) diff --git a/entities/L2Segment.cs b/entities/L2Segment.cs new file mode 100644 index 0000000..0265651 --- /dev/null +++ b/entities/L2Segment.cs @@ -0,0 +1,23 @@ +using System; +using ln.types.odb.attributes; +using System.Collections.Generic; +using ln.types.net; +namespace ln.skyscanner.entities +{ + public class L2Segment + { + [DocumentID] + public Guid ID = Guid.NewGuid(); + + public String Name; + public Network4 Network; + + public String Comment; + + public List PoPs { get; } = new List(); + + public L2Segment() + { + } + } +} diff --git a/entities/Node.cs b/entities/Node.cs index 16ab90e..f64adce 100644 --- a/entities/Node.cs +++ b/entities/Node.cs @@ -27,6 +27,7 @@ namespace ln.skyscanner.entities public String PrimaryMac { get; set; } public String Name { get; set; } + public String Comment { get; set; } public DateTime Created { get; } public DateTime LastUpdate { get; set; } diff --git a/entities/PointOfPresence.cs b/entities/PointOfPresence.cs index 8c965ac..a7e33dc 100644 --- a/entities/PointOfPresence.cs +++ b/entities/PointOfPresence.cs @@ -30,6 +30,15 @@ namespace ln.skyscanner.entities public GeoLocation GeoLocation { get; set; } public DateTime Created { get; private set; } = DateTime.Now; + [JsonIgnore] + public IEnumerable L2Segments + { + get + { + return SkyScanner.Instance.Entities.L2SegmentCollection.Query("PoPs[]", this); + } + } + public PointOfPresence() { } diff --git a/http/SkyScannerHttpManagement.cs b/http/SkyScannerHttpManagement.cs index e074985..2738749 100644 --- a/http/SkyScannerHttpManagement.cs +++ b/http/SkyScannerHttpManagement.cs @@ -11,7 +11,7 @@ using System; using ln.http.resources; using ln.logging; using System.Threading; -using sharp.logging; +using ln.logging; namespace ln.skyscanner.http { public class SkyScannerHttpManagement : JsonCallResource diff --git a/import/skytron/SkytronImport.cs b/import/skytron/SkytronImport.cs new file mode 100644 index 0000000..aad8af3 --- /dev/null +++ b/import/skytron/SkytronImport.cs @@ -0,0 +1,200 @@ +using System; +using MySql.Data.MySqlClient; +using ln.logging; +using ln.skyscanner.entities; +using System.Linq; +using ln.types.net; +using ln.types; +using System.Globalization; +namespace ln.skyscanner.import.skytron +{ + public class SkytronImport + { + MySqlConnection mysql; + + public SkytronImport(String connectionString) + { + mysql = new MySqlConnection(connectionString); + } + + public void Import() + { + try + { + mysql.Open(); + + ImportStammdaten(); + ImportNodes(); + ImportL2Segmente(); + + } catch (Exception e) + { + Logging.Log(e); + + if (mysql != null) + mysql.Close(); + + throw; + } + } + + public void ImportStammdaten() + { + MySqlCommand command = mysql.CreateCommand(); + + command.CommandText = "SELECT * FROM sender_stammdaten"; + using (MySqlDataReader reader = command.ExecuteReader()) + { + + while (reader.Read()) + { + PointOfPresence pop = SkyScanner.Instance.Entities.PointOfPresenceCollection.Query("ForeignName", reader.GetInt32("senderid").ToString()).FirstOrDefault(); + if (pop == null) + { + Logging.Log(LogLevel.INFO, "New PoP: {0} / {1}", reader.GetInt32("senderid"), reader.GetString("stationsname")); + pop = new PointOfPresence(); + pop.ForeignName = reader.GetInt32("senderid").ToString(); + pop.Name = reader.GetString("stationsname"); + + + String lati = reader.GetString("koordinaten_breite"); + String longi = reader.GetString("koordinaten_laenge"); + + if (!String.Empty.Equals(lati) && !String.Empty.Equals(longi)) + pop.GeoLocation = new GeoLocation(float.Parse(lati,CultureInfo.InvariantCulture.NumberFormat),float.Parse(longi, CultureInfo.InvariantCulture.NumberFormat)); + + SkyScanner.Instance.Entities.PointOfPresenceCollection.Insert(pop); + } + } + } + } + + public void ImportNodes() + { + MySqlCommand command = mysql.CreateCommand(); + + command.CommandText = "SELECT * FROM sender_ip_discovery"; + using (MySqlDataReader reader = command.ExecuteReader()) + { + + while (reader.Read()) + { + String id = reader.GetString("id"); + IPv4 primaryIP = IPv4.Parse(reader.GetString("ip")); + + Node node = SkyScanner.Instance.Entities.NodeCollection.Query("uniqueIdentity", id).FirstOrDefault(); + if (node == null) + { + node = new Node(primaryIP); + node.UniqueIdentity = id; + Logging.Log(LogLevel.INFO, "New Node: {0} / {1}",id,primaryIP); + } + + node.Vendor = reader.GetString("hersteller"); + node.Product = reader.GetString("produkt"); + node.Comment = reader.GetString("devicename"); + + if (reader.GetInt32("http_port") != 0) + { + node.RemoveURI("http"); + node.AddURI(new URI(String.Format("http://{0}:{1}", + primaryIP, + reader.GetString("http_port") + ))); + } + if (reader.GetInt32("ssh_port") != 0) + { + node.RemoveURI("ssh"); + node.AddURI(new URI(String.Format("ssh://{0}:{1}@{2}:{3}", + reader.GetString("ssh_user"), + reader.GetString("ssh_passwort"), + primaryIP, + reader.GetString("ssh_port") + ))); + } + if (!String.Empty.Equals(reader.GetString("snmp_community"))) + { + node.RemoveURI("snmp"); + node.AddURI(new URI(String.Format("snmp://{0}@{1}#1", + reader.GetString("snmp_community"), + primaryIP + ))); + } + if (!String.Empty.Equals(reader.GetString("snmp_v3_user"))) + { + node.RemoveURI("snmp"); + node.AddURI(new URI(String.Format("snmp://{0}:{1}:{2}@{3}#3", + reader.GetString("snmp_v3_user"), + reader.GetString("snmp_v3_auth_password"), + reader.GetString("snmp_v3_encrypt_password"), + primaryIP + ))); + } + + node.Severity = CheckSeverity.PRODUCTION; + + SkyScanner.Instance.Entities.NodeCollection.Upsert(node); + + } + } + } + + public void ImportL2Segmente() + { + MySqlCommand command = mysql.CreateCommand(); + + command.CommandText = "SELECT * FROM sender_ip_subnetze"; + + using (MySqlDataReader reader = command.ExecuteReader()) + { + while (reader.Read()) + { + Network4 net4 = Network4.Parse(reader.GetString("cidr")); + L2Segment l2 = SkyScanner.Instance.Entities.L2SegmentCollection.Query("Network", net4).FirstOrDefault(); + if (l2 == null) + { + l2 = new L2Segment(); + l2.Name = String.Format("SUB#{0}", reader.GetInt32("id").ToString()); + l2.Comment = reader.GetString("beschreibung"); + l2.Network = net4; + + Logging.Log(LogLevel.INFO, "New L2Segment: {0} / {1}",net4,l2.Name); + + String sid = reader.GetString("senderid1"); + if ((sid != null) && !String.Empty.Equals(sid)) + { + PointOfPresence pop = SkyScanner.Instance.Entities.PointOfPresenceCollection.Query("ForeignName", sid).FirstOrDefault(); + if (pop != null) + l2.PoPs.Add(pop); + } + sid = reader.GetString("senderid2"); + if ((sid != null) && !String.Empty.Equals(sid)) + { + PointOfPresence pop = SkyScanner.Instance.Entities.PointOfPresenceCollection.Query("ForeignName", sid).FirstOrDefault(); + if (pop != null) + l2.PoPs.Add(pop); + } + sid = reader.GetString("senderid3"); + if ((sid != null) && !String.Empty.Equals(sid)) + { + PointOfPresence pop = SkyScanner.Instance.Entities.PointOfPresenceCollection.Query("ForeignName", sid).FirstOrDefault(); + if (pop != null) + l2.PoPs.Add(pop); + } + sid = reader.GetString("senderid4"); + if ((sid != null) && !String.Empty.Equals(sid)) + { + PointOfPresence pop = SkyScanner.Instance.Entities.PointOfPresenceCollection.Query("ForeignName", sid).FirstOrDefault(); + if (pop != null) + l2.PoPs.Add(pop); + } + + SkyScanner.Instance.Entities.L2SegmentCollection.Insert(l2); + } + + } + } + } + + } +} diff --git a/ln.skyscanner.csproj b/ln.skyscanner.csproj index 014235a..532b13f 100644 --- a/ln.skyscanner.csproj +++ b/ln.skyscanner.csproj @@ -34,10 +34,29 @@ ..\packages\SSH.NET.2016.1.0\lib\net40\Renci.SshNet.dll - ..\packages\Newtonsoft.Json.12.0.1\lib\net45\Newtonsoft.Json.dll + ..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll + + ..\packages\BouncyCastle.1.8.3.1\lib\BouncyCastle.Crypto.dll + + + ..\packages\Google.Protobuf.3.6.1\lib\net45\Google.Protobuf.dll + + + ..\packages\MySql.Data.8.0.16\lib\net452\MySql.Data.dll + + + + + + + + + + + @@ -76,13 +95,15 @@ - + + + @@ -205,6 +226,9 @@ PreserveNewest + + PreserveNewest + @@ -222,6 +246,8 @@ + + diff --git a/packages.config b/packages.config index 78afeed..53d4781 100644 --- a/packages.config +++ b/packages.config @@ -1,5 +1,8 @@  - + + + + \ No newline at end of file diff --git a/templates/static/checks/issues.html b/templates/static/checks/issues.html index d96194a..aee0471 100644 --- a/templates/static/checks/issues.html +++ b/templates/static/checks/issues.html @@ -6,33 +6,41 @@ Warnungen:
-
- - -
-
-
-
-
-
-
-
-
-
+
+ +
-
+ +
+
+
+
+
+
+
+
+
+
 
+
+ + +

+   + [ WebUI ]
Last Check:
Current State for:
+
+
@@ -107,6 +115,8 @@ chart.data.labels.length = 0; chart.data.datasets[0].data.length = 0; chart.data.datasets[0].label = performanceValue.PerfName; + + $("#chart", htmlChart).data("chart", chart); $.each( perfData, function(){ if (this.TimeStamp != 0) @@ -118,6 +128,13 @@ function showCharts(issue) { var htmlCharts = $("#Charts"); + + $("canvas", htmlCharts).each(function(i,e){ + var htmlCanvas = $(e); + htmlCanvas.data("chart").destroy(); + htmlCanvas.data("chart",null); + }); + htmlCharts.empty(); $.each( issue.PerformanceValues, function(){ @@ -153,6 +170,17 @@ }); $("#LastCheckTime", ne).text(issue.LastCheckTime); $("#UnchangedTime", ne).text(issue.UnchangedTime); + $("#Comment", ne).text(issue.Node.Comment); + + var awebui = $("#webui", ne); + awebui.hide(); + + issue.Node.URIs.forEach(function(e,i){ + if (e.Scheme == "http"){ + awebui.show(); + awebui.attr("href", "http://" + e.Host + ":" + e.Port); + } + }, this); var histParent = $("#History", ne); histParent.empty(); @@ -201,7 +229,7 @@ currentUpdateCycle++; $.each( issueList , function(){ - if (this.Node.DeviceType != "UNKNOWN") +// if (this.Node.DeviceType != "UNKNOWN") updateIssue(this); }); diff --git a/templates/static/css/style.css b/templates/static/css/style.css index 3cf8ffc..49fc14c 100644 --- a/templates/static/css/style.css +++ b/templates/static/css/style.css @@ -17,7 +17,7 @@ div { margin: 0px; padding: 0px; - flex-grow: 0; + flex-grow: 0; } div#body { @@ -25,12 +25,12 @@ div#body { height: 100%; width: 100%; - flex-direction: column; + flex-direction: column; } h1,h2,h3,h4,h5,h6 { margin: 0px; - padding:2px; + padding:2px; } table { @@ -82,7 +82,7 @@ button { } #header > div { - margin: 4px; + margin: 4px; } @@ -130,13 +130,21 @@ button { } .scroll { - display: inline-block; - overflow: scroll; + position: absolute; + top: 0; + bottom: 0; + overflow: scroll; +} +.scroll-y { + overflow-y: scroll; +} +.scroll-x { + overflow-x: scroll; } .fill { flex-grow: 1; - flex-shrink: 1; + flex-shrink: 1; } .right { @@ -146,7 +154,7 @@ button { .flex { display: flex; - position: relative; + position: relative; } .flex.row { @@ -159,10 +167,15 @@ button { .flex > div { margin: 4px; - display: inline-block; - + display: inline-block; } +.flex > div.flex { + display: flex; +} + + + .skylogo { display: inline-block; border: 1px solid black; @@ -228,7 +241,7 @@ button { #navigation h1 { font-size: 14px; - border: none; + border: none; } #navigation > div { @@ -246,7 +259,7 @@ button { } #content > div { - margin: 6px; + margin: 6px; } @@ -327,6 +340,18 @@ fieldset > label { background-image: url(images/ui-icons_6495ED_256x240.png); } +.viewport { + position: relative; + display: block; + border: 1px dotted black; + + max-height: 100%; +} + +#issue-frame { +} + + .issue-box { display: block; diff --git a/templates/static/frame.html b/templates/static/frame.html index 4b36880..69b47ae 100644 --- a/templates/static/frame.html +++ b/templates/static/frame.html @@ -32,7 +32,7 @@ <%include "topnav.html"%>
-
+
<%=__frame__%>
diff --git a/templates/static/network/index.html b/templates/static/network/index.html index 0ef1356..b9f302a 100644 --- a/templates/static/network/index.html +++ b/templates/static/network/index.html @@ -104,7 +104,8 @@ { title: "Hersteller", data: "Vendor" }, { title: "Produkt", data: "Product" }, { title: "Gerätetyp", data: "DeviceType" }, - { title: "Standort", data: "Location", render: function(d,t,r){ return d.Latitude + "/" + d.Longitude; } } + { title: "Standort", data: "Location", render: function(d,t,r){ return d.Latitude + "/" + d.Longitude; } }, + { title: "UniqueIdentity", data: "UniqueIdentity" } ], columnDefs: [ { targets: 0, data: null, defaultContent: "" } diff --git a/templates/static/network/l2segments.html b/templates/static/network/l2segments.html new file mode 100644 index 0000000..8fb44bf --- /dev/null +++ b/templates/static/network/l2segments.html @@ -0,0 +1,107 @@ +<%frame "frame.html"%> + +

Points of Presence

+
+

List

+
+
+ +
+
+

Details

+
+
+
+
+
+
+
+
+ +
+ + diff --git a/templates/static/topnav.html b/templates/static/topnav.html index bd79a3d..b9227bc 100644 --- a/templates/static/topnav.html +++ b/templates/static/topnav.html @@ -12,7 +12,7 @@
PoP Liste