From 081d118ced62e40e6fdf2f0babb5367b4f266879 Mon Sep 17 00:00:00 2001 From: Harald Wolff Date: Thu, 21 Mar 2019 13:06:36 +0000 Subject: [PATCH] WIP --- Program.cs | 7 ++++ crawl/CrawlPool.cs | 18 ++++++---- crawl/CrawledHost.cs | 3 +- crawl/CrawledSubnet.cs | 47 ++++++++++++++++++++++++++ crawl/Crawler.cs | 32 ++++++++++++------ crawl/HostCrawl.cs | 2 ++ crawl/SubnetCrawl.cs | 18 +++++++--- crawl/tests/SNMP.cs | 16 +++++---- crawl/tests/SSH.cs | 11 +++--- entities/GlobalNetwork.cs | 21 ++++++++++++ entities/NetworkInterface.cs | 54 ++++++++++++++++++++++++++++++ entities/Node.cs | 44 +++++++++++++++++++----- entities/Router.cs | 36 -------------------- entities/Subnet.cs | 27 ++++++++++++--- http/CrawlerApi.cs | 2 +- ln.skyscanner.csproj | 4 ++- templates/static/crawlerhosts.html | 4 +-- 17 files changed, 261 insertions(+), 85 deletions(-) create mode 100644 crawl/CrawledSubnet.cs create mode 100644 entities/GlobalNetwork.cs create mode 100644 entities/NetworkInterface.cs delete mode 100644 entities/Router.cs diff --git a/Program.cs b/Program.cs index 678af51..b40be43 100644 --- a/Program.cs +++ b/Program.cs @@ -31,6 +31,7 @@ using System.Threading; using System.Net.NetworkInformation; using ln.skyscanner.crawl.tests; using Castle.DynamicProxy; +using ln.types.odb; namespace ln.skyscanner { @@ -55,6 +56,12 @@ namespace ln.skyscanner private static void Initialize() { + //ODB dB = new ODB("testdb"); + + //dB.Root.HostForIP(CIDR.Parse("1.2.3.4")); + //dB.Root.HostForIP(CIDR.Parse("2.3.4.5")); + + //dB.SavePersistent(dB.RootObject); } } diff --git a/crawl/CrawlPool.cs b/crawl/CrawlPool.cs index 9be59ff..568ea2b 100644 --- a/crawl/CrawlPool.cs +++ b/crawl/CrawlPool.cs @@ -13,12 +13,13 @@ using ln.types; using System.Net; using System.Linq; using ln.skyscanner.entities; +using ln.types.odb; namespace ln.skyscanner.crawl { - public class CrawlPool + public class CrawlPool : Persistent { private List hosts = new List(); - private List subnets = new List(); + private List subnets = new List(); public CrawlPool() { @@ -30,7 +31,7 @@ namespace ln.skyscanner.crawl } } - public Subnet[] Subnets + public CrawledSubnet[] Subnets { get { @@ -62,17 +63,22 @@ namespace ln.skyscanner.crawl } } - public Subnet GetSubnet(CIDR network) + public void RemoveHost(CrawledHost crawledHost) + { + hosts.Remove(crawledHost); + } + + public CrawledSubnet GetSubnet(CIDR network) { lock (this) { - foreach (Subnet subnet in subnets) + foreach (CrawledSubnet subnet in subnets) { if (subnet.Network.Equals(network)) return subnet; } - Subnet net = new Subnet(network); + CrawledSubnet net = new CrawledSubnet(network); subnets.Add(net); return net; } diff --git a/crawl/CrawledHost.cs b/crawl/CrawledHost.cs index c515974..744006e 100644 --- a/crawl/CrawledHost.cs +++ b/crawl/CrawledHost.cs @@ -20,10 +20,11 @@ using System.Linq; using ln.snmp; using ln.snmp.endpoint; using ln.snmp.rfc1213; +using ln.types.odb; namespace ln.skyscanner.crawl { - public class CrawledHost + public class CrawledHost : Persistent { private List interfaces = new List(); public RFC1213.Interface[] Interfaces diff --git a/crawl/CrawledSubnet.cs b/crawl/CrawledSubnet.cs new file mode 100644 index 0000000..8e61f38 --- /dev/null +++ b/crawl/CrawledSubnet.cs @@ -0,0 +1,47 @@ +// /** +// * File: Subnet.cs +// * Author: haraldwolff +// * +// * This file and it's content is copyrighted by the Author and / or copyright holder. +// * Any use wihtout proper permission is illegal and may lead to legal actions. +// * +// * +// **/ +using System; +using ln.types; +using ln.types.odb; +using System.Collections.Generic; +using System.Linq; +namespace ln.skyscanner.entities +{ + public class CrawledSubnet : Persistent + { + public readonly CIDR Network; + public String Name; + + public DateTime FirstSeen; + + public DateTime LastScan; + public DateTime NextScan; + + + private CrawledSubnet() + { + FirstSeen = DateTime.Now; + } + + public CrawledSubnet(CIDR cidr) + { + Network = cidr; + Name = Network.ToString(); + FirstSeen = DateTime.Now; + } + + + public override string ToString() + { + return Network.ToString(); + } + + } +} diff --git a/crawl/Crawler.cs b/crawl/Crawler.cs index deef558..a6490e7 100644 --- a/crawl/Crawler.cs +++ b/crawl/Crawler.cs @@ -28,6 +28,7 @@ using System.Threading; using ln.snmp.types; using Newtonsoft.Json; using Newtonsoft.Json.Converters; +using ln.types.odb; namespace ln.skyscanner.crawl { @@ -37,6 +38,7 @@ namespace ln.skyscanner.crawl public String BasePath { get; set; } public String PerfPath => Path.Combine(BasePath, "perfdb"); + public string PoolPath => Path.Combine(BasePath, "pool"); bool stopping; @@ -45,16 +47,18 @@ namespace ln.skyscanner.crawl public PoolJob[] CurrentJobs => crawlThreadPool.CurrentPoolJobs; public PoolJob[] QueuedJobs => crawlThreadPool.QueuedJobs; - public DiskObject _CrawlPool; - public CrawlPool CrawlPool => _CrawlPool?.Instance; + public CrawlPool CrawlPool => dbCrawlPool?.Root; + public ODB DBCrawlPool => dbCrawlPool; + - //public SNMPEngine SNMPEngine { get; private set; } [JsonConverter(typeof(StringEnumConverter))] public ComponentState CrawlerState { get; private set; } Thread threadScheduler; + private ODB dbCrawlPool; + public Crawler(SkyScanner skyScanner) { SkyScanner = skyScanner; @@ -83,8 +87,7 @@ namespace ln.skyscanner.crawl { stopping = false; - if (_CrawlPool == null) - _CrawlPool = new DiskObject(String.Format("{0}/pool", BasePath)); + dbCrawlPool = new ODB(PoolPath); if (crawlThreadPool == null) crawlThreadPool = new Pool(128); @@ -126,7 +129,7 @@ namespace ln.skyscanner.crawl crawlThreadPool.Close(); - _CrawlPool.Save(); + Sync(); stopping = false; CrawlerState = ComponentState.STOPPED; @@ -139,10 +142,14 @@ namespace ln.skyscanner.crawl } public void Sync() + { + Sync(CrawlPool); + } + public void Sync(IPersistent persistent) { lock (CrawlPool) { - _CrawlPool.Save(); + dbCrawlPool.SavePersistent(persistent); } } @@ -163,7 +170,7 @@ namespace ln.skyscanner.crawl } else { - Subnet subnet = CrawlPool.GetSubnet(cidr); + CrawledSubnet subnet = CrawlPool.GetSubnet(cidr); SubnetCrawl crawlSubnet = new SubnetCrawl(this, subnet); crawlThreadPool.Enqueue(crawlSubnet); } @@ -179,13 +186,18 @@ namespace ln.skyscanner.crawl { foreach (CrawledHost crawledHost in CrawlPool.Hosts) { - if (crawledHost.NextCheck < DateTime.Now) + CrawledHost crawledHost2 = CrawlPool.HostForIP(crawledHost.PrimaryIP); + + if (crawledHost != crawledHost2) + { + CrawlPool.RemoveHost(crawledHost); + } else if (crawledHost.NextCheck < DateTime.Now) { Crawl(crawledHost.PrimaryIP); } } - foreach (Subnet subnet in CrawlPool.Subnets.ToArray()) + foreach (CrawledSubnet subnet in CrawlPool.Subnets.ToArray()) { if (subnet.NextScan < (DateTime.Now - TimeSpan.FromDays(1)) && subnet.Network.MaskWidth >= 24) { diff --git a/crawl/HostCrawl.cs b/crawl/HostCrawl.cs index c70b819..7899c33 100644 --- a/crawl/HostCrawl.cs +++ b/crawl/HostCrawl.cs @@ -55,6 +55,8 @@ namespace ln.skyscanner.crawl CrawledHost.LastCheckTime = DateTime.Now - dateTime; CrawledHost.LastCheck = dateTime; CrawledHost.NextCheck = dateTime + TimeSpan.FromHours(1); + + Crawler.Sync(CrawledHost); } public override int GetHashCode() diff --git a/crawl/SubnetCrawl.cs b/crawl/SubnetCrawl.cs index d6b4a00..9a8fe64 100644 --- a/crawl/SubnetCrawl.cs +++ b/crawl/SubnetCrawl.cs @@ -15,18 +15,20 @@ using System.Net.NetworkInformation; using ln.logging; using Newtonsoft.Json; using System.Linq; +using ln.types.odb; namespace ln.skyscanner.crawl { - public class SubnetCrawl : PoolJob + public class SubnetCrawl : PoolJob, IPersistent { [JsonIgnoreAttribute] public Crawler Crawler { get; } - public Subnet Subnet { get; } + public CrawledSubnet Subnet { get; } public CIDR Network { get; } - public SubnetCrawl(Crawler crawler, Subnet subnet) + + public SubnetCrawl(Crawler crawler, CrawledSubnet subnet) { Crawler = crawler; Subnet = subnet; @@ -34,7 +36,7 @@ namespace ln.skyscanner.crawl Name = String.Format("Subnet crawl {0}", subnet.Network); } - public SubnetCrawl(Crawler crawler, Subnet subnet,CIDR network) + public SubnetCrawl(Crawler crawler, CrawledSubnet subnet,CIDR network) { Crawler = crawler; Subnet = subnet; @@ -99,6 +101,9 @@ namespace ln.skyscanner.crawl Subnet.LastScan = startTime; Subnet.NextScan = startTime + TimeSpan.FromHours(24); } + + Crawler.Sync(Subnet); + } public override int GetHashCode() @@ -116,5 +121,10 @@ namespace ln.skyscanner.crawl } + private Guid PersistenceID = Guid.NewGuid(); + public Guid GetPersistenceID() + { + return PersistenceID; + } } } diff --git a/crawl/tests/SNMP.cs b/crawl/tests/SNMP.cs index f8d14a9..351c47a 100644 --- a/crawl/tests/SNMP.cs +++ b/crawl/tests/SNMP.cs @@ -24,7 +24,7 @@ namespace ln.skyscanner.crawl.tests { if (!TestCurrentHints(crawledHost)) { - + return TestDefaults(crawledHost); } return false; @@ -228,11 +228,15 @@ namespace ln.skyscanner.crawl.tests v2.CommunityString = crawledHost.GetHint("snmp.community"); return v2; case 3: - USMEndpoint endpoint = new USMEndpoint(SNMPEngine.DefaultEngine, new System.Net.IPEndPoint(snmpAddress, 161)); - endpoint.AuthMethod = SnmpV3AuthMethod.SHA; - endpoint.Username = crawledHost.GetHint("snmp.username"); - endpoint.AuthKeyPhrase = crawledHost.GetHint("snmp.authkey"); - return endpoint; + if (crawledHost.GetHint("snmp.username", null) != null) + { + USMEndpoint endpoint = new USMEndpoint(SNMPEngine.DefaultEngine, new System.Net.IPEndPoint(snmpAddress, 161)); + endpoint.AuthMethod = SnmpV3AuthMethod.SHA; + endpoint.Username = crawledHost.GetHint("snmp.username"); + endpoint.AuthKeyPhrase = crawledHost.GetHint("snmp.authkey"); + return endpoint; + } + return null; default: return null; } diff --git a/crawl/tests/SSH.cs b/crawl/tests/SSH.cs index b6eaf3a..55c3e4f 100644 --- a/crawl/tests/SSH.cs +++ b/crawl/tests/SSH.cs @@ -41,7 +41,7 @@ namespace ln.skyscanner.crawl.tests string sshUser = crawledHost.GetHint("ssh.login", null); string sshPassword = crawledHost.GetHint("ssh.password", null); - if ((sshPort == -1) || !CanConnect(crawledHost,sshIP.Host.ToString(),sshPort,sshUser,sshPassword)) + if ((sshPort == -1) || !CanConnect(crawledHost,sshIP.Host,sshPort,sshUser,sshPassword)) { if (!Scan(crawledHost)) { @@ -64,7 +64,7 @@ namespace ln.skyscanner.crawl.tests { foreach (string password in defaultPasswords) { - if (CanConnect(crawledHost, ip.Host.ToString(), port, "skytron", password)) + if (CanConnect(crawledHost, ip.Host, port, "skytron", password)) return true; } } @@ -72,9 +72,9 @@ namespace ln.skyscanner.crawl.tests return false; } - private static bool CanConnect(CrawledHost crawledHost, String host, int port, string username, string password) + private static bool CanConnect(CrawledHost crawledHost, CIDR host, int port, string username, string password) { - using (SshClient client = new SshClient(host, port, username, password)) + using (SshClient client = new SshClient(host.ToString(), port, username, password)) { client.ConnectionInfo.Timeout = TimeSpan.FromSeconds(1); try @@ -82,7 +82,7 @@ namespace ln.skyscanner.crawl.tests client.Connect(); crawledHost.SetHint("ssh.port", client.ConnectionInfo.Port); - crawledHost.SetHint("ssh.ip", client.ConnectionInfo.Host); + crawledHost.SetHint("ssh.ip", host); crawledHost.SetHint("ssh.login", client.ConnectionInfo.Username); crawledHost.SetHint("ssh.password", password); crawledHost.SetHint("ssh.version", client.ConnectionInfo.ServerVersion); @@ -95,6 +95,7 @@ namespace ln.skyscanner.crawl.tests } catch (SocketException) { + return false; } } return false; diff --git a/entities/GlobalNetwork.cs b/entities/GlobalNetwork.cs new file mode 100644 index 0000000..1c87bec --- /dev/null +++ b/entities/GlobalNetwork.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Collections; +using System.Security; + +namespace ln.skyscanner.entities +{ + public class GlobalNetwork + { + public IEnumerable Subnets => subnets; + public IEnumerable Nodes => nodes; + + + private List nodes = new List(); + private List subnets = new List(); + + public GlobalNetwork() + { + } + } +} diff --git a/entities/NetworkInterface.cs b/entities/NetworkInterface.cs new file mode 100644 index 0000000..14dfabf --- /dev/null +++ b/entities/NetworkInterface.cs @@ -0,0 +1,54 @@ +// /** +// * File: Router.cs +// * Author: haraldwolff +// * +// * This file and it's content is copyrighted by the Author and / or copyright holder. +// * Any use wihtout proper permission is illegal and may lead to legal actions. +// * +// * +// **/ +using System; +using System.Collections.Generic; +using ln.types; +using ln.skyscanner.crawl; +namespace ln.skyscanner.entities +{ + public class NetworkInterface + { + public Node Node { get; private set; } = null; + public string Name { get; private set; } = ""; + + public CIDR[] IPs { get; set; } = new CIDR[0]; + + private NetworkInterface() + { } + + public NetworkInterface(Node node) + { + Node = node; + } + public NetworkInterface(Node node,String name) + { + Node = node; + Name = name; + } + + public override int GetHashCode() + { + return Node.GetHashCode() ^ Name.GetHashCode(); + } + + public override bool Equals(object obj) + { + if (obj is NetworkInterface) + { + NetworkInterface networkInterface = obj as NetworkInterface; + return Node.Equals(networkInterface.Node) && Name.Equals(networkInterface.Name); + } + return false; + } + + + } + +} diff --git a/entities/Node.cs b/entities/Node.cs index e926c63..67c9472 100644 --- a/entities/Node.cs +++ b/entities/Node.cs @@ -10,27 +10,53 @@ using System; using ln.types; using System.Collections.Generic; +using System.Linq; namespace ln.skyscanner.entities { public class Node { - public CIDR PrimaryIP; - public String Name; + public GlobalNetwork GlobalNetwork { get; private set; } - public List NetworkInterfaces = new List(); + public CIDR PrimaryIP { get; set; } + public String PrimaryMac { get; set; } + public String Name { get; set; } - public DateTime FirstSeen; - public DateTime LastSeen; + public DateTime Created { get; } + public Uri[] URIs => uris.ToArray(); - public Node() + + private HashSet uris = new HashSet(); + + private Node() { - FirstSeen = DateTime.Now; } - public Node(CIDR primaryIP) + public Node(GlobalNetwork globalNetwork, CIDR primaryIP) { + GlobalNetwork = globalNetwork; + PrimaryIP = primaryIP; - FirstSeen = DateTime.Now; + Name = PrimaryIP.ToString(); + Created = DateTime.Now; + } + + public void RemoveURI(Uri uri) + { + foreach (Uri u in uris) + { + if (uri.Equals(u)) + uris.Remove(u); + } + } + + public void AddURI(Uri uri) + { + uris.Add(uri); + } + + public IEnumerable FindURIs(string scheme) + { + return uris.Where((u) => scheme.Equals(u.Scheme)); } } diff --git a/entities/Router.cs b/entities/Router.cs deleted file mode 100644 index dc1885f..0000000 --- a/entities/Router.cs +++ /dev/null @@ -1,36 +0,0 @@ -// /** -// * File: Router.cs -// * Author: haraldwolff -// * -// * This file and it's content is copyrighted by the Author and / or copyright holder. -// * Any use wihtout proper permission is illegal and may lead to legal actions. -// * -// * -// **/ -using System; -using System.Collections.Generic; -using ln.types; -namespace ln.skyscanner.entities -{ - public class NetworkInterface - { - public Node Node; - public string Name; - public List IPs = new List(); - - public NetworkInterface() - { } - - public NetworkInterface(Node node) - { - Node = node; - } - public NetworkInterface(Node node,String name) - { - Node = node; - Name = name; - } - - } - -} diff --git a/entities/Subnet.cs b/entities/Subnet.cs index 56bf997..74d88c0 100644 --- a/entities/Subnet.cs +++ b/entities/Subnet.cs @@ -9,10 +9,15 @@ // **/ using System; using ln.types; +using ln.types.odb; +using System.Collections.Generic; +using System.Linq; namespace ln.skyscanner.entities { - public class Subnet + public class Subnet : Persistent { + public GlobalNetwork GlobalNetwork { get; private set; } + public readonly CIDR Network; public String Name; @@ -21,18 +26,32 @@ namespace ln.skyscanner.entities public DateTime LastScan; public DateTime NextScan; - public Subnet() + public NetworkInterface[] AttachedInterfaces => networkInterfaces.ToArray(); + + private HashSet networkInterfaces = new HashSet(); + + private Subnet() { - FirstSeen = DateTime.Now; } - public Subnet(CIDR cidr) + public Subnet(GlobalNetwork globalNetwork,CIDR cidr) { + GlobalNetwork = globalNetwork; Network = cidr; Name = Network.ToString(); FirstSeen = DateTime.Now; } + public void AttachInterface(NetworkInterface networkInterface) + { + networkInterfaces.Add(networkInterface); + } + public void DetachInterface(NetworkInterface networkInterface) + { + networkInterfaces.Remove(networkInterface); + } + + public override string ToString() { return Network.ToString(); diff --git a/http/CrawlerApi.cs b/http/CrawlerApi.cs index 4396de5..ab31917 100644 --- a/http/CrawlerApi.cs +++ b/http/CrawlerApi.cs @@ -20,7 +20,7 @@ namespace ln.skyscanner.http } [Callable] - public Subnet[] GetSubnets() + public CrawledSubnet[] GetSubnets() { return SkyScanner.Crawler?.CrawlPool.Subnets.ToArray(); } diff --git a/ln.skyscanner.csproj b/ln.skyscanner.csproj index a8c400d..64e9d29 100644 --- a/ln.skyscanner.csproj +++ b/ln.skyscanner.csproj @@ -50,7 +50,7 @@ - + @@ -66,6 +66,8 @@ + + diff --git a/templates/static/crawlerhosts.html b/templates/static/crawlerhosts.html index fe4cc22..fa1d2e6 100644 --- a/templates/static/crawlerhosts.html +++ b/templates/static/crawlerhosts.html @@ -117,8 +117,8 @@ { title: "Zuletzt gesehen", data: "LastSeen", sorter: "", width: 180, formatter: "datetime", formatterParams: { inputFormat: "", outputFormat: "DD.MM.YYYY hh:mm:ss", invalidPlaceHolder: "-" } }, { title: "Letzter Scan", data: "LastCheck", sorter: "", width: 180, formatter: "datetime", formatterParams: { inputFormat: "", outputFormat: "DD.MM.YYYY hh:mm:ss", invalidPlaceHolder: "-" } }, { title: "Nächster Scan",data: "NextCheck",sorter: "",width: 180,formatter: "datetime",formatterParams: {inputFormat: "",outputFormat: "DD.MM.YYYY hh:mm:ss",invalidPlaceHolder: "-"} }, - { title: "SNMP", data: "SSHDetected", width: 90 }, - { title: "SSH", data: "SnmpDetected", width: 90 }, + { title: "SSH", data: "SSHDetected", width: 90 }, + { title: "SNMP", data: "SnmpDetected", width: 90 }, { title: "RFC1213", data: "RFC1213Detected", width: 90 } ];