WIP

Program.cs

@@ -31,6 +31,7 @@ using System.Threading;
 using System.Net.NetworkInformation;
 using ln.skyscanner.crawl.tests;
 using Castle.DynamicProxy;
+using ln.types.odb;
 
 namespace ln.skyscanner
 {
@@ -55,6 +56,12 @@ namespace ln.skyscanner
 
         private static void Initialize()
         {
+            //ODB<CrawlPool> dB = new ODB<CrawlPool>("testdb");
+
+            //dB.Root.HostForIP(CIDR.Parse("1.2.3.4"));
+            //dB.Root.HostForIP(CIDR.Parse("2.3.4.5"));
+
+            //dB.SavePersistent(dB.RootObject);
         }
 
     }

crawl/CrawlPool.cs

@@ -13,12 +13,13 @@ using ln.types;
 using System.Net;
 using System.Linq;
 using ln.skyscanner.entities;
+using ln.types.odb;
 namespace ln.skyscanner.crawl
 {
-    public class CrawlPool
+    public class CrawlPool : Persistent
     {
         private List<CrawledHost> hosts = new List<CrawledHost>();
-        private List<Subnet> subnets = new List<Subnet>();
+        private List<CrawledSubnet> subnets = new List<CrawledSubnet>();
 
         public CrawlPool()
         {
@@ -30,7 +31,7 @@ namespace ln.skyscanner.crawl
             }
         }
 
-        public Subnet[] Subnets
+        public CrawledSubnet[] Subnets
         {
             get
             {
@@ -62,17 +63,22 @@ namespace ln.skyscanner.crawl
             }
         }
 
-        public Subnet GetSubnet(CIDR network)
+        public void RemoveHost(CrawledHost crawledHost)
+        {
+            hosts.Remove(crawledHost);
+        }
+
+        public CrawledSubnet GetSubnet(CIDR network)
         {
             lock (this)
             {
-                foreach (Subnet subnet in subnets)
+                foreach (CrawledSubnet subnet in subnets)
                 {
                     if (subnet.Network.Equals(network))
                         return subnet;
                 }
 
-                Subnet net = new Subnet(network);
+                CrawledSubnet net = new CrawledSubnet(network);
                 subnets.Add(net);
                 return net;
             }

crawl/CrawledHost.cs

@@ -20,10 +20,11 @@ using System.Linq;
 using ln.snmp;
 using ln.snmp.endpoint;
 using ln.snmp.rfc1213;
+using ln.types.odb;
 
 namespace ln.skyscanner.crawl
 {
-    public class CrawledHost
+    public class CrawledHost : Persistent
     {
         private List<RFC1213.Interface> interfaces = new List<RFC1213.Interface>();
         public RFC1213.Interface[] Interfaces

crawl/CrawledSubnet.cs

@@ -0,0 +1,47 @@
+// /**
+//  * File: Subnet.cs
+//  * Author: haraldwolff
+//  *
+//  * This file and it's content is copyrighted by the Author and / or copyright holder.
+//  * Any use wihtout proper permission is illegal and may lead to legal actions.
+//  *
+//  *
+//  **/
+using System;
+using ln.types;
+using ln.types.odb;
+using System.Collections.Generic;
+using System.Linq;
+namespace ln.skyscanner.entities
+{
+    public class CrawledSubnet : Persistent
+    {
+        public readonly CIDR Network;
+        public String Name;
+
+        public DateTime FirstSeen;
+
+        public DateTime LastScan;
+        public DateTime NextScan;
+
+
+        private CrawledSubnet()
+        {
+            FirstSeen = DateTime.Now;
+        }
+
+        public CrawledSubnet(CIDR cidr)
+        {
+            Network = cidr;
+            Name = Network.ToString();
+            FirstSeen = DateTime.Now;
+        }
+
+
+        public override string ToString()
+        {
+            return Network.ToString();
+        }
+
+    }
+}

crawl/Crawler.cs

@@ -28,6 +28,7 @@ using System.Threading;
 using ln.snmp.types;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Converters;
+using ln.types.odb;
 
 namespace ln.skyscanner.crawl
 {
@@ -37,6 +38,7 @@ namespace ln.skyscanner.crawl
 
         public String BasePath { get; set; }
         public String PerfPath => Path.Combine(BasePath, "perfdb");
+        public string PoolPath => Path.Combine(BasePath, "pool");
 
         bool stopping;
 
@@ -45,16 +47,18 @@ namespace ln.skyscanner.crawl
         public PoolJob[] CurrentJobs => crawlThreadPool.CurrentPoolJobs;
         public PoolJob[] QueuedJobs => crawlThreadPool.QueuedJobs;
 
-        public DiskObject<CrawlPool> _CrawlPool;
+        public CrawlPool CrawlPool => dbCrawlPool?.Root;
-        public CrawlPool CrawlPool => _CrawlPool?.Instance;
+        public ODB<CrawlPool> DBCrawlPool => dbCrawlPool;
+
 
-        //public SNMPEngine SNMPEngine { get; private set; }
 
         [JsonConverter(typeof(StringEnumConverter))]
         public ComponentState CrawlerState { get; private set; }
 
         Thread threadScheduler;
 
+        private ODB<CrawlPool> dbCrawlPool;
+
         public Crawler(SkyScanner skyScanner)
         {
             SkyScanner = skyScanner;
@@ -83,8 +87,7 @@ namespace ln.skyscanner.crawl
             {
                 stopping = false;
 
-                if (_CrawlPool == null)
+                dbCrawlPool = new ODB<CrawlPool>(PoolPath);
-                    _CrawlPool = new DiskObject<CrawlPool>(String.Format("{0}/pool", BasePath));
 
                 if (crawlThreadPool == null)
                     crawlThreadPool = new Pool(128);
@@ -126,7 +129,7 @@ namespace ln.skyscanner.crawl
 
                     crawlThreadPool.Close();
 
-                    _CrawlPool.Save();
+                    Sync();
 
                     stopping = false;
                     CrawlerState = ComponentState.STOPPED;
@@ -139,10 +142,14 @@ namespace ln.skyscanner.crawl
         }
 
         public void Sync()
+        {
+            Sync(CrawlPool);
+        }
+        public void Sync(IPersistent persistent)
         {
             lock (CrawlPool)
             {
-                _CrawlPool.Save();
+                dbCrawlPool.SavePersistent(persistent);
             }
         }
 
@@ -163,7 +170,7 @@ namespace ln.skyscanner.crawl
                 }
                 else
                 {
-                    Subnet subnet = CrawlPool.GetSubnet(cidr);
+                    CrawledSubnet subnet = CrawlPool.GetSubnet(cidr);
                     SubnetCrawl crawlSubnet = new SubnetCrawl(this, subnet);
                     crawlThreadPool.Enqueue(crawlSubnet);
                 }
@@ -179,13 +186,18 @@ namespace ln.skyscanner.crawl
             {
                 foreach (CrawledHost crawledHost in CrawlPool.Hosts)
                 {
-                    if (crawledHost.NextCheck < DateTime.Now)
+                    CrawledHost crawledHost2 = CrawlPool.HostForIP(crawledHost.PrimaryIP);
+
+                    if (crawledHost != crawledHost2)
+                    {
+                        CrawlPool.RemoveHost(crawledHost);
+                    } else if (crawledHost.NextCheck < DateTime.Now)
                     {
                         Crawl(crawledHost.PrimaryIP);
                     }
                 }
 
-                foreach (Subnet subnet in CrawlPool.Subnets.ToArray())
+                foreach (CrawledSubnet subnet in CrawlPool.Subnets.ToArray())
                 {
                     if (subnet.NextScan < (DateTime.Now - TimeSpan.FromDays(1)) && subnet.Network.MaskWidth >= 24)
                     {

crawl/HostCrawl.cs

@@ -55,6 +55,8 @@ namespace ln.skyscanner.crawl
             CrawledHost.LastCheckTime = DateTime.Now - dateTime;
             CrawledHost.LastCheck = dateTime;
             CrawledHost.NextCheck = dateTime + TimeSpan.FromHours(1);
+
+            Crawler.Sync(CrawledHost);
         }
 
         public override int GetHashCode()

crawl/SubnetCrawl.cs

@@ -15,18 +15,20 @@ using System.Net.NetworkInformation;
 using ln.logging;
 using Newtonsoft.Json;
 using System.Linq;
+using ln.types.odb;
 namespace ln.skyscanner.crawl
 {
-    public class SubnetCrawl : PoolJob
+    public class SubnetCrawl : PoolJob, IPersistent
     {
         [JsonIgnoreAttribute]
         public Crawler Crawler { get; }
-        public Subnet Subnet { get; }
+        public CrawledSubnet Subnet { get; }
 
         public CIDR Network { get; }
 
 
-        public SubnetCrawl(Crawler crawler, Subnet subnet)
+
+        public SubnetCrawl(Crawler crawler, CrawledSubnet subnet)
         {
             Crawler = crawler;
             Subnet = subnet;
@@ -34,7 +36,7 @@ namespace ln.skyscanner.crawl
 
             Name = String.Format("Subnet crawl {0}", subnet.Network);
         }
-        public SubnetCrawl(Crawler crawler, Subnet subnet,CIDR network)
+        public SubnetCrawl(Crawler crawler, CrawledSubnet subnet,CIDR network)
         {
             Crawler = crawler;
             Subnet = subnet;
@@ -99,6 +101,9 @@ namespace ln.skyscanner.crawl
                 Subnet.LastScan = startTime;
                 Subnet.NextScan = startTime + TimeSpan.FromHours(24);
             }
+
+            Crawler.Sync(Subnet);
+
         }
 
         public override int GetHashCode()
@@ -116,5 +121,10 @@ namespace ln.skyscanner.crawl
         }
 
 
+        private Guid PersistenceID = Guid.NewGuid();
+        public Guid GetPersistenceID()
+        {
+            return PersistenceID;
+        }
     }
 }

crawl/tests/SNMP.cs

@@ -24,7 +24,7 @@ namespace ln.skyscanner.crawl.tests
         {
             if (!TestCurrentHints(crawledHost))
             {
-
+                return TestDefaults(crawledHost);
             }
 
             return false;
@@ -228,11 +228,15 @@ namespace ln.skyscanner.crawl.tests
                     v2.CommunityString = crawledHost.GetHint<string>("snmp.community");
                     return v2;
                 case 3:
-                    USMEndpoint endpoint = new USMEndpoint(SNMPEngine.DefaultEngine, new System.Net.IPEndPoint(snmpAddress, 161));
+                    if (crawledHost.GetHint<string>("snmp.username", null) != null)
-                    endpoint.AuthMethod = SnmpV3AuthMethod.SHA;
+                    {
-                    endpoint.Username = crawledHost.GetHint<string>("snmp.username");
+                        USMEndpoint endpoint = new USMEndpoint(SNMPEngine.DefaultEngine, new System.Net.IPEndPoint(snmpAddress, 161));
-                    endpoint.AuthKeyPhrase = crawledHost.GetHint<string>("snmp.authkey");
+                        endpoint.AuthMethod = SnmpV3AuthMethod.SHA;
-                    return endpoint;
+                        endpoint.Username = crawledHost.GetHint<string>("snmp.username");
+                        endpoint.AuthKeyPhrase = crawledHost.GetHint<string>("snmp.authkey");
+                        return endpoint;
+                    }
+                    return null;
                 default:
                     return null;
             }

crawl/tests/SSH.cs

@@ -41,7 +41,7 @@ namespace ln.skyscanner.crawl.tests
             string sshUser = crawledHost.GetHint<string>("ssh.login", null);
             string sshPassword = crawledHost.GetHint<string>("ssh.password", null);
 
-            if ((sshPort == -1) || !CanConnect(crawledHost,sshIP.Host.ToString(),sshPort,sshUser,sshPassword))
+            if ((sshPort == -1) || !CanConnect(crawledHost,sshIP.Host,sshPort,sshUser,sshPassword))
             {
                 if (!Scan(crawledHost))
                 {
@@ -64,7 +64,7 @@ namespace ln.skyscanner.crawl.tests
                 {
                     foreach (string password in defaultPasswords)
                     {
-                        if (CanConnect(crawledHost, ip.Host.ToString(), port, "skytron", password))
+                        if (CanConnect(crawledHost, ip.Host, port, "skytron", password))
                             return true;
                     }
                 }
@@ -72,9 +72,9 @@ namespace ln.skyscanner.crawl.tests
             return false;
         }
 
-        private static bool CanConnect(CrawledHost crawledHost, String host, int port, string username, string password)
+        private static bool CanConnect(CrawledHost crawledHost, CIDR host, int port, string username, string password)
         {
-            using (SshClient client = new SshClient(host, port, username, password))
+            using (SshClient client = new SshClient(host.ToString(), port, username, password))
             {
                 client.ConnectionInfo.Timeout = TimeSpan.FromSeconds(1);
                 try
@@ -82,7 +82,7 @@ namespace ln.skyscanner.crawl.tests
                     client.Connect();
 
                     crawledHost.SetHint("ssh.port", client.ConnectionInfo.Port);
-                    crawledHost.SetHint("ssh.ip", client.ConnectionInfo.Host);
+                    crawledHost.SetHint("ssh.ip", host);
                     crawledHost.SetHint("ssh.login", client.ConnectionInfo.Username);
                     crawledHost.SetHint("ssh.password", password);
                     crawledHost.SetHint("ssh.version", client.ConnectionInfo.ServerVersion);
@@ -95,6 +95,7 @@ namespace ln.skyscanner.crawl.tests
                 }
                 catch (SocketException)
                 {
+                    return false;
                 }
             }
             return false;

entities/GlobalNetwork.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Collections;
+using System.Security;
+
+namespace ln.skyscanner.entities
+{
+    public class GlobalNetwork
+    {
+        public IEnumerable<Subnet> Subnets => subnets;
+        public IEnumerable<Node> Nodes => nodes;
+
+
+        private List<Node> nodes = new List<Node>();
+        private List<Subnet> subnets = new List<Subnet>();
+
+        public GlobalNetwork()
+        {
+        }
+    }
+}

entities/NetworkInterface.cs

@@ -0,0 +1,54 @@
+// /**
+//  * File: Router.cs
+//  * Author: haraldwolff
+//  *
+//  * This file and it's content is copyrighted by the Author and / or copyright holder.
+//  * Any use wihtout proper permission is illegal and may lead to legal actions.
+//  *
+//  *
+//  **/
+using System;
+using System.Collections.Generic;
+using ln.types;
+using ln.skyscanner.crawl;
+namespace ln.skyscanner.entities
+{
+    public class NetworkInterface
+    {
+        public Node Node { get; private set; } = null;
+        public string Name { get; private set; } = "";
+
+        public CIDR[] IPs { get; set; } = new CIDR[0];
+
+        private NetworkInterface()
+        { }
+
+        public NetworkInterface(Node node)
+        {
+            Node = node;
+        }
+        public NetworkInterface(Node node,String name)
+        {
+            Node = node;
+            Name = name;
+        }
+
+        public override int GetHashCode()
+        {
+            return Node.GetHashCode() ^ Name.GetHashCode();
+        }
+
+        public override bool Equals(object obj)
+        {
+            if (obj is NetworkInterface)
+            {
+                NetworkInterface networkInterface = obj as NetworkInterface;
+                return Node.Equals(networkInterface.Node) && Name.Equals(networkInterface.Name);
+            }
+            return false;
+        }
+
+
+    }
+
+}

entities/Node.cs

@@ -10,27 +10,53 @@
 using System;
 using ln.types;
 using System.Collections.Generic;
+using System.Linq;
 namespace ln.skyscanner.entities
 {
     public class Node
     {
-        public CIDR PrimaryIP;
+        public GlobalNetwork GlobalNetwork { get; private set; }
-        public String Name;
 
-        public List<NetworkInterface> NetworkInterfaces = new List<NetworkInterface>();
+        public CIDR PrimaryIP { get; set; } 
+        public String PrimaryMac { get; set; }
+        public String Name { get; set; }
 
-        public DateTime FirstSeen;
+        public DateTime Created { get; }
-        public DateTime LastSeen;
+        public Uri[] URIs => uris.ToArray();
 
-        public Node()
+
+        private HashSet<Uri> uris = new HashSet<Uri>();
+
+        private Node()
         {
-            FirstSeen = DateTime.Now;
         }
 
-        public Node(CIDR primaryIP)
+        public Node(GlobalNetwork globalNetwork, CIDR primaryIP)
         {
+            GlobalNetwork = globalNetwork;
+
             PrimaryIP = primaryIP;
-            FirstSeen = DateTime.Now;
+            Name = PrimaryIP.ToString();
+            Created = DateTime.Now;
+        }
+
+        public void RemoveURI(Uri uri)
+        {
+            foreach (Uri u in uris)
+            {
+                if (uri.Equals(u))
+                    uris.Remove(u);
+            }
+        }
+
+        public void AddURI(Uri uri)
+        {
+            uris.Add(uri);
+        }
+
+        public IEnumerable<Uri> FindURIs(string scheme)
+        {
+            return uris.Where((u) => scheme.Equals(u.Scheme));
         }
 
     }

entities/Router.cs

@@ -1,36 +0,0 @@
-// /**
-//  * File: Router.cs
-//  * Author: haraldwolff
-//  *
-//  * This file and it's content is copyrighted by the Author and / or copyright holder.
-//  * Any use wihtout proper permission is illegal and may lead to legal actions.
-//  *
-//  *
-//  **/
-using System;
-using System.Collections.Generic;
-using ln.types;
-namespace ln.skyscanner.entities
-{
-    public class NetworkInterface
-    {
-        public Node Node;
-        public string Name;
-        public List<CIDR> IPs = new List<CIDR>();
-
-        public NetworkInterface()
-        { }
-
-        public NetworkInterface(Node node)
-        {
-            Node = node;
-        }
-        public NetworkInterface(Node node,String name)
-        {
-            Node = node;
-            Name = name;
-        }
-
-    }
-
-}

entities/Subnet.cs

@@ -9,10 +9,15 @@
 //  **/
 using System;
 using ln.types;
+using ln.types.odb;
+using System.Collections.Generic;
+using System.Linq;
 namespace ln.skyscanner.entities
 {
-    public class Subnet
+    public class Subnet : Persistent
     {
+        public GlobalNetwork GlobalNetwork { get; private set; }
+
         public readonly CIDR Network;
         public String Name;
 
@@ -21,18 +26,32 @@ namespace ln.skyscanner.entities
         public DateTime LastScan;
         public DateTime NextScan;
 
-        public Subnet()
+        public NetworkInterface[] AttachedInterfaces => networkInterfaces.ToArray();
+
+        private HashSet<NetworkInterface> networkInterfaces = new HashSet<NetworkInterface>();
+
+        private Subnet()
         {
-            FirstSeen = DateTime.Now;
         }
 
-        public Subnet(CIDR cidr)
+        public Subnet(GlobalNetwork globalNetwork,CIDR cidr)
         {
+            GlobalNetwork = globalNetwork;
             Network = cidr;
             Name = Network.ToString();
             FirstSeen = DateTime.Now;
         }
 
+        public void AttachInterface(NetworkInterface networkInterface)
+        {
+            networkInterfaces.Add(networkInterface);
+        }
+        public void DetachInterface(NetworkInterface networkInterface)
+        {
+            networkInterfaces.Remove(networkInterface);
+        }
+
+
         public override string ToString()
         {
             return Network.ToString();

http/CrawlerApi.cs

@@ -20,7 +20,7 @@ namespace ln.skyscanner.http
         }
 
         [Callable]
-        public Subnet[] GetSubnets()
+        public CrawledSubnet[] GetSubnets()
         {
             return SkyScanner.Crawler?.CrawlPool.Subnets.ToArray();
         }

ln.skyscanner.csproj

@@ -50,7 +50,7 @@
     <Compile Include="crawl\Crawler.cs" />
     <Compile Include="crawl\CrawledHost.cs" />
     <Compile Include="entities\Subnet.cs" />
-    <Compile Include="entities\Router.cs" />
+    <Compile Include="entities\NetworkInterface.cs" />
     <Compile Include="Check.cs" />
     <Compile Include="check\Hostalive.cs" />
     <Compile Include="SkyScanner.cs" />
@@ -66,6 +66,8 @@
     <Compile Include="crawl\tests\SNMP.cs" />
     <Compile Include="crawl\tests\ICMP.cs" />
     <Compile Include="crawl\tests\RFC1213.cs" />
+    <Compile Include="entities\GlobalNetwork.cs" />
+    <Compile Include="crawl\CrawledSubnet.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="packages.config" />

templates/static/crawlerhosts.html

@@ -117,8 +117,8 @@
         { title: "Zuletzt gesehen", data: "LastSeen", sorter: "", width: 180, formatter: "datetime", formatterParams: { inputFormat: "", outputFormat: "DD.MM.YYYY hh:mm:ss", invalidPlaceHolder: "-" } },
         { title: "Letzter Scan", data: "LastCheck", sorter: "", width: 180, formatter: "datetime", formatterParams: { inputFormat: "", outputFormat: "DD.MM.YYYY hh:mm:ss", invalidPlaceHolder: "-" } },
         { title: "Nächster Scan",data: "NextCheck",sorter: "",width: 180,formatter: "datetime",formatterParams: {inputFormat: "",outputFormat: "DD.MM.YYYY hh:mm:ss",invalidPlaceHolder: "-"} },
-        { title: "SNMP", data: "SSHDetected", width: 90 },
+        { title: "SSH", data: "SSHDetected", width: 90 },
-        { title: "SSH", data: "SnmpDetected", width: 90 },
+        { title: "SNMP", data: "SnmpDetected", width: 90 },
         { title: "RFC1213", data: "RFC1213Detected", width: 90 }
     ];