WIP

Program.cs

@@ -31,7 +31,7 @@ namespace ln.skyscanner
             fileLogger.MaxLogLevel = LogLevel.INFO;
             Logger.Default.Backends.Add(fileLogger);
 
-            //Logger.ConsoleLogger.MaxLogLevel = LogLevel.DEBUGDETAIL;
+            Logger.ConsoleLogger.MaxLogLevel = LogLevel.INFO;
 
             Initialize();
 

SkyScanner.cs

@@ -11,6 +11,7 @@ using sharp.logging;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Converters;
 using ln.skyscanner.checks;
+using System.Collections.Generic;
 namespace ln.skyscanner
 {
     [JsonConverter(typeof(StringEnumConverter))]
@@ -40,7 +41,7 @@ namespace ln.skyscanner
 
             Logging.Log(LogLevel.INFO, "SkyScanner: Constructor");
 
-            BasePath = Path.GetFullPath(".");
+            BasePath = Path.GetFullPath("/var/cache/ln.skyscanner");
 
             Logging.Log(LogLevel.INFO, "SkyScanner: BasePath={0}", BasePath);
 
@@ -48,6 +49,19 @@ namespace ln.skyscanner
 
             Arguments = args;
 
+            Queue<string> qArguments = new Queue<string>(Arguments);
+            while (qArguments.Count > 0)
+            {
+                string arg = qArguments.Dequeue();
+                switch (arg)
+                {
+                    case "-p":
+                        BasePath = Path.GetFullPath(qArguments.Dequeue());
+                        break;
+                }
+            }
+
+
             Entities = new SkyEntities(this);
             Checker = new SkyChecker();
         }

checks/Hostalive.cs

@@ -26,6 +26,7 @@ namespace ln.skyscanner.checks
             Ping = new Ping();
         }
 
+        public override bool IsCritical => true;
         public override bool IsValid(Node node) => node.PrimaryIP != null;
 
         public override void Check(SkyChecker skyChecker, Node node)

checks/SkyCheck.cs

@@ -38,6 +38,7 @@ namespace ln.skyscanner.checks
         static SkyCheck()
         {
             AddSkyCheck(new Hostalive());
+            AddSkyCheck(new Ubiquity());
         }
     }
 }

checks/SkyChecker.cs

@@ -16,6 +16,7 @@ using ln.types.threads;
 using System.Threading;
 using System.Linq;
 using ln.logging;
+using ln.snmp;
 namespace ln.skyscanner.checks
 {
     public class SkyChecker
@@ -28,6 +29,8 @@ namespace ln.skyscanner.checks
         public string[] PerfNames => perfFiles.Keys.ToArray();
         public bool ContainsPerfFile(String perfName) => perfFiles.ContainsKey(perfName);
 
+        public SNMPEngine SNMPEngine { get; }
+
         BTree<string, PerfFile> perfFiles = new BTree<string, PerfFile>();
 
         Pool checkPool = new Pool(0);
@@ -38,6 +41,7 @@ namespace ln.skyscanner.checks
         public SkyChecker()
         {
             BasePath = Path.Combine(SkyScanner.Instance.BasePath, "perfdb");
+            SNMPEngine = new SNMPEngine();
         }
 
         public void Start()

checks/Ubiquity.cs

@@ -0,0 +1,54 @@
+using System;
+using ln.skyscanner.entities;
+using ln.types;
+using ln.snmp;
+using System.Collections.Generic;
+using ln.snmp.types;
+using ln.logging;
+
+namespace ln.skyscanner.checks
+{
+    public class Ubiquity : SkyCheck
+    {
+        public Ubiquity()
+            :base("ubiquity")
+        {
+        }
+
+
+        public override void Check(SkyChecker skyChecker, Node node)
+        {
+            foreach (URI snmpUri in node.FindURIs("snmp"))
+            {
+                using (SnmpInterface snmp = SnmpInterface.FromURI(snmpUri,skyChecker.SNMPEngine))
+                {
+                    Sequence[][] ptp = snmp.snmpWalk(new string[] {
+                                                    "1.3.6.1.4.1.41112.1.3.2.1.5",
+                                                    "1.3.6.1.4.1.41112.1.3.2.1.6",
+                                                    "1.3.6.1.4.1.41112.1.3.2.1.11",
+                                                    "1.3.6.1.4.1.41112.1.3.2.1.14",
+                                                    "1.3.6.1.4.1.41112.1.3.3.1.64",
+                                                    "1.3.6.1.4.1.41112.1.3.3.1.66"
+                        });
+
+                    foreach (Sequence[] row in ptp)
+                    {
+                        skyChecker.WritePerfValue(this, "ptp_rx_capa", node, (double)((Integer)(row[0].Items[1])).LongValue);
+                        skyChecker.WritePerfValue(this, "ptp_tx_capa", node, (double)((Integer)(row[1].Items[1])).LongValue);
+                        skyChecker.WritePerfValue(this, "ptp_rx_pwr", node, (double)((Integer)(row[2].Items[1])).LongValue);
+                        skyChecker.WritePerfValue(this, "ptp_tx_pwr", node, (double)((Integer)(row[3].Items[1])).LongValue);
+
+                        skyChecker.WritePerfValue(this, "ptp_rx_rate", node, (double)((Integer)(row[4].Items[1])).LongValue);   // ToDo: multiply 8 / delta T
+                        skyChecker.WritePerfValue(this, "ptp_tx_rate", node, (double)((Integer)(row[5].Items[1])).LongValue);   // ToDo: multiply 8 / delta T
+                    }
+
+                }
+            }
+        }
+
+        public override bool IsValid(Node node)
+        {
+            return (node.Vendor != null) && node.Vendor.Equals("Ubiquity");
+        }
+    }
+}

crawl/Crawler.cs

@@ -33,6 +33,7 @@ namespace ln.skyscanner.crawl
             CrawlService.RegisterService(new RFC1213());
             CrawlService.RegisterService(new HTTP());
             CrawlService.RegisterService(new SSH());
+            CrawlService.RegisterService(new Ubiquity());
         }
 
         public SkyScanner SkyScanner { get; }
@@ -85,6 +86,9 @@ namespace ln.skyscanner.crawl
                 subnets = odbDatabase.GetCollection<CrawledSubnet>();
                 blockedNetworks = odbDatabase.GetCollection<Network4>("blockedNetworks");
 
+                hosts.EnsureIndex("PrimaryIP");
+                hosts.EnsureIndex("IPAddresses[]");
+
                 CrawlerState = ComponentState.INITIALIZED;
             }
             catch (Exception)
@@ -190,27 +194,38 @@ namespace ln.skyscanner.crawl
 
         public CrawledHost FindHostForIP(IPv4 ip)
         {
-            CrawledHost crawledHost = CrawledHosts.Where(host => host.HasIP(ip)).FirstOrDefault();
+            lock (this)
-            if (crawledHost == null)
             {
-                crawledHost = new CrawledHost();
+                Query nodeByIpQuery = Query.OR(
-                crawledHost.PrimaryIP = ip;
+                    Query.Equals<CrawledHost>("IPAddresses[]", ip),
-                crawledHost.Name = ip.ToString();
+                    Query.Equals<CrawledHost>("PrimaryIP", ip)
+                    );
+                CrawledHost crawledHost = hosts.Select(nodeByIpQuery).FirstOrDefault();
+                if (crawledHost == null)
+                {
+                    crawledHost = new CrawledHost();
+                    crawledHost.PrimaryIP = ip;
+                    crawledHost.Name = ip.ToString();
 
-                CrawledHosts.Insert(crawledHost);
+                    CrawledHosts.Insert(crawledHost);
+                }
+                return crawledHost;
             }
-            return crawledHost;
         }
         public CrawledSubnet FindSubnet(Network4 network)
         {
-            CrawledSubnet sn = CrawledSubnets.Select("Network", network).FirstOrDefault();
+            lock (this)
-            if (sn == null)
             {
-                Logging.Log(LogLevel.INFO, "Crawler adds new subnet: {0}",network);
+                Query subnetQuery = Query.Equals<CrawledSubnet>("Network", network);
-                sn = new CrawledSubnet(network);
+                CrawledSubnet sn = CrawledSubnets.Select(subnetQuery).FirstOrDefault();
-                CrawledSubnets.Insert(sn);
+                if (sn == null)
+                {
+                    Logging.Log(LogLevel.INFO, "Crawler adds new subnet: {0}", network);
+                    sn = new CrawledSubnet(network);
+                    CrawledSubnets.Insert(sn);
+                }
+                return sn;
             }
-            return sn;
         }
 
 

crawl/service/ICMP.cs

@@ -50,6 +50,8 @@ namespace ln.skyscanner.crawl.tests
                     crawl.Host.SetHint("ping.out_of_ten", nSuccess);
 
                     crawl.Host.LastSeen = DateTime.Now;
+                    if (crawl.Host.FirstSeen < new DateTime(1970, 1, 2))
+                        crawl.Host.FirstSeen = DateTime.Now;
                 }
                 else
                 {

crawl/service/RFC1213.cs

@@ -15,6 +15,7 @@ using ln.types;
 using ln.skyscanner.crawl.service;
 using System.Configuration;
 using System.Net.Sockets;
+using ln.types.net;
 namespace ln.skyscanner.crawl.tests
 {
     public class RFC1213 : CrawlService
@@ -48,6 +49,7 @@ namespace ln.skyscanner.crawl.tests
                         {
                             crawl.Host.IPAddresses = interfaces.SelectMany(intf => intf.IPAddresses).ToArray();
                             crawl.Host.HWAddresses = interfaces.Select(intf => intf.HWAddr).ToArray();
+                            crawl.Host.Networks = interfaces.SelectMany(intf => intf.IPAddresses).Zip(interfaces.SelectMany(intf => intf.Netmasks), (ip, nm) => new Network4(ip.AsUInt, nm.AsUInt)).ToArray();
 
                             crawl.Host.SetHint("rfc1213.interfaces", interfaces.Select(intf => string.Format("{0};{1};{2}",
                                                                                                                          intf.Name,

crawl/service/SNMP.cs

@@ -38,6 +38,7 @@ namespace ln.skyscanner.crawl.tests
             {
                 TestDefaults(crawl.Host);
             }
+
             return true;
         }
 

entities/GlobalNetwork.cs

@@ -238,6 +238,27 @@ namespace ln.skyscanner.entities
                     );
             }
 
+            node.RemoveURI("snmp");
+            if (crawledHost.GetHint("snmp.version", -1) == 3)
+            {
+                node.AddURI(new URI(String.Format("snmp://{0}:{1}:{2}@{3}#{4}",
+                    crawledHost.GetHint("snmp.username",""),
+                    crawledHost.GetHint("snmp.authkey",""),
+                    crawledHost.GetHint("snmp.privkey",""),
+                    crawledHost.GetHint<IPv4>("snmp.address"),
+                    crawledHost.GetHint<int>("snmp.version")
+                    )));
+            }
+            else if (crawledHost.GetHint("snmp.version", -1) > 0)
+            {
+                node.AddURI(new URI(String.Format("snmp://{0}@{1}#{2}",
+                    crawledHost.GetHint("snmp.community", ""),
+                    crawledHost.GetHint<IPv4>("snmp.address"),
+                    crawledHost.GetHint<int>("snmp.version")
+                    )));
+
+            }
+
 
             node.RemoveURI("http");
             if (crawledHost.GetHint("http.port",-1) != -1)

http/CrawlerApi.cs

@@ -50,6 +50,16 @@ namespace ln.skyscanner.http
             return host;
         }
 
+        [Callable]
+        public void TriggerRecrawl()
+        {
+            foreach (CrawledHost host in Crawler.CrawledHosts.ToArray())
+            {
+                host.NextCheck = DateTime.Now;
+                Crawler.CrawledHosts.Upsert(host);
+            }
+        }
+
         [Callable]
         public void Crawl(string _id)
         {

ln.skyscanner.csproj

@@ -82,6 +82,7 @@
     <Compile Include="checks\SkyCheck.cs" />
     <Compile Include="checks\CheckJob.cs" />
     <Compile Include="http\CheckerApi.cs" />
+    <Compile Include="checks\Ubiquity.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="packages.config" />

templates/static/checks/index.html

@@ -27,6 +27,13 @@
             },
             options: {
                 scales: {
+                    yAxes: [
+                        {
+                            ticks: {
+                                callback: ScaleSI
+                            }
+                        }
+                    ],
                     xAxes: [{
                         type: 'time',
                         time: {

templates/static/crawlerhosts.html

@@ -5,6 +5,7 @@
 </div>
 
 <div>
+    <button onclick="skyapi().call('api/crawler','TriggerRecrawl');">Force Recrawl</button>
     <button onclick="refreshHostTable();">!?</button>
     <div>
         Bezeichnung: <input type="text" id="htName" />

templates/static/network/index.html

@@ -60,7 +60,7 @@
             $("#dLocation").prop("disabled", false).val(node.Location.Latitude + " / " + node.Location.Longitude);
     
             $("#dURIs").empty();
-            $.each( node.URIs, function(){ $("#dURIs").append($("<span>" + this.Scheme + "://" + this.Host + ":" + this.Port + "</span><br>")); } );
+            $.each( node.URIs, function(){ $("#dURIs").append($("<span>" + this.Scheme + "://" + this.Host + ":" + this.Port + (this.Fragment ? "#" + this.Fragment : "") + "</span><br>")); } );
     
             $("#dInterfaces").empty();
             $.each( node.Interfaces, function(){

templates/static/skyapi.js

@@ -176,3 +176,18 @@ function skyapi()
 {
     return __skyapi;
 }
+
+
+
+function ScaleSI(value)
+{
+    if (value > 1000000000)
+        return ((value / 1000000000) | 0) + "G";
+    if (value > 1000000)
+        return ((value / 1000000) | 0) + "M";
+    if (value > 1000)
+        return ((value / 1000) | 0) + "k";
+    return value;
+}
+
+