From e124f5e8855a5119d9f79b432b2e3fea4b8a77f3 Mon Sep 17 00:00:00 2001 From: Harald Wolff Date: Thu, 21 Mar 2019 07:43:32 +0100 Subject: [PATCH] WIP --- Program.cs | 87 +------ SkyScanner.cs | 2 + crawl/CrawlHost.cs | 312 ----------------------- crawl/CrawlTests.cs | 33 +++ crawl/CrawledHost.cs | 43 +--- crawl/Crawler.cs | 16 +- crawl/HostCrawl.cs | 75 ++++++ crawl/{CrawlSubnet.cs => SubnetCrawl.cs} | 12 +- crawl/tests/Crawling.cs | 64 +++++ crawl/tests/ICMP.cs | 62 +++++ crawl/tests/RFC1213.cs | 39 +++ crawl/tests/SNMP.cs | 244 ++++++++++++++++++ crawl/tests/SSH.cs | 106 ++++++++ ln.skyscanner.csproj | 14 +- packages.config | 1 + templates/static/crawlerhosts.html | 14 +- 16 files changed, 678 insertions(+), 446 deletions(-) delete mode 100644 crawl/CrawlHost.cs create mode 100644 crawl/CrawlTests.cs create mode 100644 crawl/HostCrawl.cs rename crawl/{CrawlSubnet.cs => SubnetCrawl.cs} (90%) create mode 100644 crawl/tests/Crawling.cs create mode 100644 crawl/tests/ICMP.cs create mode 100644 crawl/tests/RFC1213.cs create mode 100644 crawl/tests/SNMP.cs create mode 100644 crawl/tests/SSH.cs diff --git a/Program.cs b/Program.cs index bfcdec3..678af51 100644 --- a/Program.cs +++ b/Program.cs @@ -29,25 +29,14 @@ using ln.types.sync; using ln.skyscanner.crawl; using System.Threading; using System.Net.NetworkInformation; +using ln.skyscanner.crawl.tests; +using Castle.DynamicProxy; namespace ln.skyscanner { class MainClass { - private static void DumpPerfValues(PerfValue[] perfValues) - { - int n = 0; - - Logging.Log("----------------------------------------------"); - Logging.Log("Dumping {0} perfValues:", perfValues.Length); - foreach (PerfValue perfValue in perfValues) - { - Logging.Log("PerfValue: [{1,6}] {0}", perfValue, n); - n++; - } - Logging.Log(""); - } public static void Main(string[] args) { @@ -57,81 +46,15 @@ namespace ln.skyscanner Logger.ConsoleLogger.MaxLogLevel = LogLevel.INFO; + Initialize(); + SkyScanner skyScanner = new SkyScanner(args); skyScanner.Start(); } - public static void TestAuthKey(String filename,USMEndpoint v3endpoint) + private static void Initialize() { - FileStream fileStream = new FileStream(filename, FileMode.Open); - byte[] source = new byte[fileStream.Length]; - - int nread = fileStream.Read(source, 0, source.Length); - - fileStream.Close(); - fileStream.Dispose(); - - ASN1Value asn = new ASN1Value(source); - USMMessage usm = new USMMessage(asn); - - byte[] repro1 = asn.AsByteArray; - byte[] repro2 = ((ASN1Value)usm).AsByteArray; - - Logging.Log(LogLevel.DEBUG, "Source: {0}", BitConverter.ToString(source)); - Logging.Log(LogLevel.DEBUG, "Repro1: {0}", BitConverter.ToString(repro1)); - Logging.Log(LogLevel.DEBUG, "Repro2: {0}", BitConverter.ToString(repro2)); - - if (!source.SequenceEqual(repro1)) - Logging.Log(LogLevel.ERROR, "Repro1 does not match!"); - else - Logging.Log(LogLevel.ERROR, "Repro1 matches!"); - - if (!source.SequenceEqual(repro2)) - Logging.Log(LogLevel.ERROR, "Repro2 does not match!"); - else - Logging.Log(LogLevel.ERROR, "Repro2 matches!"); - - if (!repro1.SequenceEqual(repro2)) - Logging.Log(LogLevel.ERROR, "Repro1 != Repro2!"); - else - Logging.Log(LogLevel.ERROR, "Repro1/2 match!"); - - - - usm.Dump(); - usm.SecurityParameters.Dump(); - - byte[] auth1 = usm.SecurityParameters.msgAuthenticationParameters.Bytes; - - usm.SecurityParameters.msgAuthenticationParameters.Bytes = new byte[12]; - - Logging.Log(LogLevel.DEBUG, "Source: {0}", BitConverter.ToString(source)); - - v3endpoint.RemoteEngineID = usm.SecurityParameters.msgAuthoritativeEngineID; - v3endpoint.CacheAuthoritativeEngineTime = usm.SecurityParameters.msgAuthoritativeEngineTime; - v3endpoint.CacheAuthoritativeEngineBoots = usm.SecurityParameters.msgAuthoritativeEngineBoots; - - v3endpoint.LocalizeKeys(); - - v3endpoint.AuthenticateMessage(usm); - - byte[] auth2 = usm.SecurityParameters.msgAuthenticationParameters.Bytes; - - Logging.Log(LogLevel.DEBUG, "Authenticated: {0}", BitConverter.ToString(((ASN1Value)usm).AsByteArray)); - - Logging.Log(LogLevel.DEBUG, "Original Auth Token: {0}", BitConverter.ToString(auth1)); - Logging.Log(LogLevel.DEBUG, "Calculated Auth Token: {0}", BitConverter.ToString(auth2)); - - if (auth1.SequenceEqual(auth2)) - { - Logging.Log(LogLevel.DEBUG, "MATCH"); - } - else - { - Logging.Log(LogLevel.DEBUG, "NO MATCH"); - } - } } diff --git a/SkyScanner.cs b/SkyScanner.cs index 4885967..593eb1f 100644 --- a/SkyScanner.cs +++ b/SkyScanner.cs @@ -43,6 +43,8 @@ namespace ln.skyscanner Arguments = args; } + public SkyScanner() + { } public void Start() diff --git a/crawl/CrawlHost.cs b/crawl/CrawlHost.cs deleted file mode 100644 index 89fc6c7..0000000 --- a/crawl/CrawlHost.cs +++ /dev/null @@ -1,312 +0,0 @@ -// /** -// * File: CrawlHost.cs -// * Author: haraldwolff -// * -// * This file and it's content is copyrighted by the Author and / or copyright holder. -// * Any use wihtout proper permission is illegal and may lead to legal actions. -// * -// * -// **/ -using System; -using ln.types.threads; -using ln.logging; -using System.Net.NetworkInformation; -using ln.snmp.endpoint; -using ln.snmp; -using System.Net; -using ln.snmp.types; -using System.Collections.Generic; -using ln.snmp.rfc1213; -using ln.types; -using Newtonsoft.Json; -namespace ln.skyscanner.crawl -{ - public class CrawlHost : PoolJob - { - [JsonIgnore] - public Crawler Crawler { get; } - [JsonIgnore] - public CrawledHost CrawledHost { get; } - - public CrawlHost(Crawler crawler,CrawledHost crawledHost) - { - Crawler = crawler; - CrawledHost = crawledHost; - - Name = String.Format("Host crawl {0} [ {1} ]", crawledHost.Name, crawledHost.PrimaryIP); - } - - public override void RunJob() - { - DateTime dateTime = DateTime.Now; - - if (crawlPing(CrawledHost)) - { - if (CrawledHost.FirstSeen.Equals(DateTime.MinValue)) - CrawledHost.FirstSeen = DateTime.Now; - - CrawledHost.LastSeen = DateTime.Now; - - try - { - - CrawlSNMP(CrawledHost); - - - } catch (Exception e) - { - Logging.Log(LogLevel.ERROR, "CrawlHost: {0}: caught exception",CrawledHost.PrimaryIP); - Logging.Log(e); - } - - - } - else - { - - } - - CrawledHost.LastCheck = dateTime; - CrawledHost.NextCheck = dateTime + TimeSpan.FromHours(1); - } - - public bool crawlPing(CrawledHost crawledHost) - { - using (Ping ping = new Ping()) - { - setState("ICMP check"); - - int nSuccess = 0; - long roundTripTime = 0; - - for (int n = 0; n < 10; n++) - { - setState("ICMP check [{0}/10]",n); - - PingReply pingReply = ping.Send(crawledHost.PrimaryIP, 500); - if (pingReply.Status == IPStatus.Success) - { - nSuccess++; - roundTripTime += pingReply.RoundtripTime; - } - else if ((n > 3) && (nSuccess == 0)) - { - break; - } - } - - if (nSuccess > 0) - { - roundTripTime /= nSuccess; - - crawledHost.SetHint("ping.success", true); - crawledHost.SetHint("ping.rta", (int)roundTripTime); - crawledHost.SetHint("ping.out_of_ten", nSuccess); - Logging.Log(LogLevel.INFO, "Host is reachable: {0} RTA={1}ms", crawledHost.PrimaryIP, roundTripTime); - } - else - { - crawledHost.SetHint("ping.success", false); - crawledHost.SetHint("ping.rta", null); - crawledHost.SetHint("ping.out_of_ten", 0); - - Logging.Log(LogLevel.INFO, "Host is unreachable: {0}", crawledHost.PrimaryIP); - } - return crawledHost.GetHint("ping.success"); - } - } - - public void CrawlSNMP(CrawledHost crawledHost) - { - string[] communities = new string[] { "VhclfC7lfIojYZ", "Vhclf(C7$lfIojYZ", "ByFR4oW98hap", "qVy3hnZJ2fov" }; - - bool snmpDetected = false; - - using (USMEndpoint v3endpoint = new USMEndpoint(Crawler.SNMPEngine, new IPEndPoint(crawledHost.PrimaryIP, 161))) - { - setState("SNMPv3 check"); - try - { - v3endpoint.QueryEngineID(); - } - catch (TimeoutException) - { - } - - if (v3endpoint.RemoteEngineID != null) - { - crawledHost.SetHint("snmp.version", 3); - Logging.Log(LogLevel.INFO, "{0}: SNMPv3 support detected", crawledHost.PrimaryIP); - - bool replied = false; - int c = 0; - - foreach (string community in communities) - { - c++; - setState("SNMPv3 check [{0}/{1}]",c,communities.Length); - - v3endpoint.Username = "skytron"; - v3endpoint.AuthMethod = SnmpV3AuthMethod.SHA; - v3endpoint.AuthKeyPhrase = community; - - try - { - Variable prID = v3endpoint.snmpGet("1.3.6.1.2.1.1.2.0"); - crawledHost.SetHint("snmp.username", "skytron"); - crawledHost.SetHint("snmp.authkey", community); - crawledHost.SetHint("snmp.sysObjectID", (prID as ObjectIdentifier).AsString); - - replied = true; - break; - } - catch (TimeoutException) - { - } - } - - if (replied) - { - snmpDetected = true; - } - - } - } - - if (!snmpDetected) - { - using (SnmpV2Endpoint v2endpoint = new SnmpV2Endpoint(Crawler.SNMPEngine, new IPEndPoint(crawledHost.PrimaryIP, 161))) - { - setState("SNMPv2c check"); - - foreach (String community in communities) - { - v2endpoint.CommunityString = community; - try - { - Variable prID = v2endpoint.snmpGet("1.3.6.1.2.1.1.2.0"); - - crawledHost.SetHint("snmp.version", 2); - crawledHost.SetHint("snmp.community", community); - crawledHost.SetHint("snmp.sysObjectID", (prID as ObjectIdentifier).AsString); - - snmpDetected = true; - break; - } - catch (SnmpError) - { - } - catch (TimeoutException) - { - } - - } - } - } - - if (!snmpDetected) - { - using (SnmpV1Endpoint v1endpoint = new SnmpV1Endpoint(Crawler.SNMPEngine, new IPEndPoint(crawledHost.PrimaryIP, 161))) - { - setState("SNMPv1 check"); - - foreach (String community in communities) - { - v1endpoint.CommunityString = community; - try - { - Variable prID = v1endpoint.snmpGet("1.3.6.1.2.1.1.2.0"); - - crawledHost.SetHint("snmp.version", 1); - crawledHost.SetHint("snmp.community", community); - crawledHost.SetHint("snmp.sysObjectID", (prID as ObjectIdentifier).AsString); - - snmpDetected = true; - break; - } - catch (SnmpError) - { - } - catch (TimeoutException) - { - } - - } - } - } - - if (!snmpDetected) - { - crawledHost.SetHint("snmp.version", null); - crawledHost.SetHint("snmp.username", null); - crawledHost.SetHint("snmp.authkey", null); - crawledHost.SetHint("snmp.community", null); - crawledHost.SetHint("snmp.sysObjectID", null); - } - else - { - setState("SNMP crawl"); - try - { - using (SnmpEndpoint endpoint = crawledHost.GetSnmpEndpoint(Crawler.SNMPEngine)) - { - try - { - Sequence[] VorIDs = endpoint.snmpWalk("1.3.6.1.2.1.1.9.1.2").ToArray(); - string[] orids = new string[VorIDs.Length]; - - for (int n = 0; n < orids.Length; n++) - { - orids[n] = (VorIDs[n].Items[1] as ObjectIdentifier).AsString; - } - - crawledHost.SetHint("snmp.orids", orids); - } - catch (TimeoutException) - { } - - List ids = new List(crawledHost.GetHint("snmp.orids", new string[0])); - - if (ids.Contains("1.3.6.1.2.1.31") || crawledHost.GetHint("snmp.sysObjectID", "").Equals("1.3.6.1.4.1.14988.1")) - { - CrawlRFC1213(crawledHost, endpoint); - } - } - } - catch (Exception e) - { - Logging.Log(LogLevel.ERROR, "CrawlHost {0} caught exception: {1}", CrawledHost, e); - Logging.Log(e); - } - } - - } - - private void CrawlRFC1213(CrawledHost crawledHost, SnmpEndpoint endpoint) - { - RFC1213.Interface[] interfaces = RFC1213.GetInterfaces(endpoint); - crawledHost.Interfaces = interfaces; - - foreach (CIDR ip in crawledHost.IPAddresses) - { - if (ip.MaskWidth != 32) - Crawler.CrawlPool.GetSubnet(ip.Network); - } - } - - public override int GetHashCode() - { - return CrawledHost.GetHashCode(); - } - - public override bool Equals(object obj) - { - if (obj is CrawlHost) - { - CrawlHost you = obj as CrawlHost; - return Crawler.Equals(you.Crawler) && CrawledHost.Equals(you.CrawledHost); - } - return false; - } - } -} diff --git a/crawl/CrawlTests.cs b/crawl/CrawlTests.cs new file mode 100644 index 0000000..457db19 --- /dev/null +++ b/crawl/CrawlTests.cs @@ -0,0 +1,33 @@ +// /** +// * File: CrawlTests.cs +// * Author: haraldwolff +// * +// * This file and it's content is copyrighted by the Author and / or copyright holder. +// * Any use wihtout proper permission is illegal and may lead to legal actions. +// * +// * +// **/ +using System; +using ln.skyscanner.crawl.tests; +using System.Collections.Generic; +namespace ln.skyscanner.crawl +{ + public class CrawlTests + { + public CrawlTest[] Tests => crawlTests.ToArray(); + + + private List crawlTests = new List(); + + public CrawlTests() + { + } + + public void AddTest(CrawlTest crawlTest) + { + } + + + + } +} diff --git a/crawl/CrawledHost.cs b/crawl/CrawledHost.cs index 37049a4..c515974 100644 --- a/crawl/CrawledHost.cs +++ b/crawl/CrawledHost.cs @@ -20,6 +20,7 @@ using System.Linq; using ln.snmp; using ln.snmp.endpoint; using ln.snmp.rfc1213; + namespace ln.skyscanner.crawl { public class CrawledHost @@ -30,13 +31,22 @@ namespace ln.skyscanner.crawl get => interfaces.ToArray(); set => interfaces = new List(value); } - public CIDR[] IPAddresses => interfaces.Select((intf) => intf.IPAddresses).SelectMany((i) => i).ToArray(); public String[] HWAddresses => interfaces.Select((intf) => intf.HWAddr).ToArray(); public String PrimaryHWAddr => HWAddresses.Where((hwa) => hwa != null && !String.Empty.Equals(hwa)).FirstOrDefault(); public String Name { get; set; } public CIDR PrimaryIP { get; set; } + public CIDR[] IPAddresses + { + get + { + CIDR[] result = interfaces.Select((intf) => intf.IPAddresses).SelectMany((i) => i).ToArray(); + if (result.Length == 0) + return new CIDR[] { PrimaryIP }; + return result; + } + } Dictionary hints = new Dictionary(); @@ -45,9 +55,11 @@ namespace ln.skyscanner.crawl public DateTime LastSeen; public DateTime LastCheck; public DateTime NextCheck; + public TimeSpan LastCheckTime; public bool SnmpDetected => GetHint("snmp.version", -1) != -1; - + public bool SSHDetected => GetHint("ssh.port", -1) != -1; + public bool RFC1213Detected => GetHint("rfc1213", false); public CrawledHost() { @@ -90,33 +102,6 @@ namespace ln.skyscanner.crawl return hints.ContainsKey(name) && (hints[name] != null); } - public SnmpEndpoint GetSnmpEndpoint(SNMPEngine engine) - { - int snmpVersion = GetHint("snmp.version", -1); - - switch (snmpVersion) - { - case 1: - SnmpV1Endpoint v1 = new SnmpV1Endpoint(engine, new IPEndPoint(PrimaryIP, 161)); - v1.CommunityString = GetHint("snmp.community"); - return v1; - case 2: - SnmpV2Endpoint v2 = new SnmpV2Endpoint(engine, new IPEndPoint(PrimaryIP, 161)); - v2.CommunityString = GetHint("snmp.community"); - return v2; - case 3: - USMEndpoint endpoint = new USMEndpoint(engine, new IPEndPoint(PrimaryIP, 161)); - endpoint.AuthMethod = SnmpV3AuthMethod.SHA; - endpoint.Username = GetHint("snmp.username"); - endpoint.AuthKeyPhrase = GetHint("snmp.authkey"); - return endpoint; - default: - return null; - } - } - - - public override string ToString() { return String.Format("[CrawledHost PrimaryIP={0} Name={1}]",PrimaryIP,Name); diff --git a/crawl/Crawler.cs b/crawl/Crawler.cs index c25bfc8..deef558 100644 --- a/crawl/Crawler.cs +++ b/crawl/Crawler.cs @@ -48,7 +48,7 @@ namespace ln.skyscanner.crawl public DiskObject _CrawlPool; public CrawlPool CrawlPool => _CrawlPool?.Instance; - public SNMPEngine SNMPEngine { get; private set; } + //public SNMPEngine SNMPEngine { get; private set; } [JsonConverter(typeof(StringEnumConverter))] public ComponentState CrawlerState { get; private set; } @@ -83,11 +83,6 @@ namespace ln.skyscanner.crawl { stopping = false; - if (SNMPEngine == null) - SNMPEngine = new SNMPEngine(); - - SNMPEngine.Timeout = 1250; - if (_CrawlPool == null) _CrawlPool = new DiskObject(String.Format("{0}/pool", BasePath)); @@ -131,9 +126,6 @@ namespace ln.skyscanner.crawl crawlThreadPool.Close(); - SNMPEngine.Close(); - SNMPEngine = null; - _CrawlPool.Save(); stopping = false; @@ -166,13 +158,13 @@ namespace ln.skyscanner.crawl if (cidr.MaskWidth == 32) { CrawledHost crawledHost = CrawlPool.HostForIP(cidr); - CrawlHost crawlHost = new CrawlHost(this, crawledHost); + HostCrawl crawlHost = new HostCrawl(this, crawledHost); crawlThreadPool.Enqueue(crawlHost); } else { Subnet subnet = CrawlPool.GetSubnet(cidr); - CrawlSubnet crawlSubnet = new CrawlSubnet(this, subnet); + SubnetCrawl crawlSubnet = new SubnetCrawl(this, subnet); crawlThreadPool.Enqueue(crawlSubnet); } } catch (Exception e) @@ -195,7 +187,7 @@ namespace ln.skyscanner.crawl foreach (Subnet subnet in CrawlPool.Subnets.ToArray()) { - if (subnet.NextScan < (DateTime.Now - TimeSpan.FromDays(1))) + if (subnet.NextScan < (DateTime.Now - TimeSpan.FromDays(1)) && subnet.Network.MaskWidth >= 24) { Crawl(subnet.Network); } diff --git a/crawl/HostCrawl.cs b/crawl/HostCrawl.cs new file mode 100644 index 0000000..c70b819 --- /dev/null +++ b/crawl/HostCrawl.cs @@ -0,0 +1,75 @@ +// /** +// * File: CrawlHost.cs +// * Author: haraldwolff +// * +// * This file and it's content is copyrighted by the Author and / or copyright holder. +// * Any use wihtout proper permission is illegal and may lead to legal actions. +// * +// * +// **/ +using System; +using ln.types.threads; +using ln.logging; +using System.Net.NetworkInformation; +using ln.snmp.endpoint; +using ln.snmp; +using System.Net; +using ln.snmp.types; +using System.Collections.Generic; +using ln.snmp.rfc1213; +using ln.types; +using Newtonsoft.Json; +using ln.skyscanner.crawl.tests; +using System.Runtime.Remoting.Messaging; +namespace ln.skyscanner.crawl +{ + public class HostCrawl : PoolJob + { + [JsonIgnore] + public Crawler Crawler { get; } + [JsonIgnore] + public CrawledHost CrawledHost { get; } + + public HostCrawl(Crawler crawler,CrawledHost crawledHost) + { + Crawler = crawler; + CrawledHost = crawledHost; + + Name = String.Format("Host crawl {0} [ {1} ]", crawledHost.Name, crawledHost.PrimaryIP); + } + + public override void RunJob() + { + State = "Prepare"; + + DateTime dateTime = DateTime.Now; + + if (Crawling.Crawl(this)) + { + if (CrawledHost.FirstSeen.Equals(DateTime.MinValue)) + CrawledHost.FirstSeen = DateTime.Now; + + CrawledHost.LastSeen = DateTime.Now; + } + + CrawledHost.LastCheckTime = DateTime.Now - dateTime; + CrawledHost.LastCheck = dateTime; + CrawledHost.NextCheck = dateTime + TimeSpan.FromHours(1); + } + + public override int GetHashCode() + { + return CrawledHost.GetHashCode(); + } + + public override bool Equals(object obj) + { + if (obj is HostCrawl) + { + HostCrawl you = obj as HostCrawl; + return Crawler.Equals(you.Crawler) && CrawledHost.Equals(you.CrawledHost); + } + return false; + } + } +} diff --git a/crawl/CrawlSubnet.cs b/crawl/SubnetCrawl.cs similarity index 90% rename from crawl/CrawlSubnet.cs rename to crawl/SubnetCrawl.cs index 95a14b8..d6b4a00 100644 --- a/crawl/CrawlSubnet.cs +++ b/crawl/SubnetCrawl.cs @@ -17,7 +17,7 @@ using Newtonsoft.Json; using System.Linq; namespace ln.skyscanner.crawl { - public class CrawlSubnet : PoolJob + public class SubnetCrawl : PoolJob { [JsonIgnoreAttribute] public Crawler Crawler { get; } @@ -26,7 +26,7 @@ namespace ln.skyscanner.crawl public CIDR Network { get; } - public CrawlSubnet(Crawler crawler, Subnet subnet) + public SubnetCrawl(Crawler crawler, Subnet subnet) { Crawler = crawler; Subnet = subnet; @@ -34,7 +34,7 @@ namespace ln.skyscanner.crawl Name = String.Format("Subnet crawl {0}", subnet.Network); } - public CrawlSubnet(Crawler crawler, Subnet subnet,CIDR network) + public SubnetCrawl(Crawler crawler, Subnet subnet,CIDR network) { Crawler = crawler; Subnet = subnet; @@ -47,7 +47,7 @@ namespace ln.skyscanner.crawl { if (Network.MaskWidth < 26) { - SplitJob(Network.Divide(26 - Network.MaskWidth).Select((n) => new CrawlSubnet(Crawler, Subnet, n)).ToArray()); + SplitJob(Network.Divide(26 - Network.MaskWidth).Select((n) => new SubnetCrawl(Crawler, Subnet, n)).ToArray()); return; } @@ -107,9 +107,9 @@ namespace ln.skyscanner.crawl } public override bool Equals(object obj) { - if (obj is CrawlSubnet) + if (obj is SubnetCrawl) { - CrawlSubnet you = obj as CrawlSubnet; + SubnetCrawl you = obj as SubnetCrawl; return Crawler.Equals(you.Crawler) && Subnet.Equals(you.Subnet) && Network.Equals(you.Network); } return false; diff --git a/crawl/tests/Crawling.cs b/crawl/tests/Crawling.cs new file mode 100644 index 0000000..ab7ee05 --- /dev/null +++ b/crawl/tests/Crawling.cs @@ -0,0 +1,64 @@ +// /** +// * File: CrawlTest.cs +// * Author: haraldwolff +// * +// * This file and it's content is copyrighted by the Author and / or copyright holder. +// * Any use wihtout proper permission is illegal and may lead to legal actions. +// * +// * +// **/ +using System; +using System.Collections.Generic; +using System.Reflection; +using ln.types; + +namespace ln.skyscanner.crawl.tests +{ + public static class Crawling + { + + public static bool Crawl(HostCrawl hostCrawl) + { + hostCrawl.setState("ICMP"); + if (!ICMP.IsReachable(hostCrawl.CrawledHost)) + { + return false; + } + + if (hostCrawl.AbortRequested) + return false; + + hostCrawl.setState("SSH"); + if (SSH.CanConnect(hostCrawl.CrawledHost)) + { + // ToDo: Extract more details... + } + + if (hostCrawl.AbortRequested) + return false; + + hostCrawl.setState("SNMP"); + if (SNMP.HasSNMP(hostCrawl.CrawledHost)) + { + if (hostCrawl.AbortRequested) + return false; + + hostCrawl.setState("RFC1213"); + RFC1213.Check(hostCrawl.CrawledHost); + } + + + + + foreach (CIDR ip in hostCrawl.CrawledHost.IPAddresses) + { + if (ip.MaskWidth != 32) + hostCrawl.Crawler.CrawlPool.GetSubnet(ip.Network); + } + + return true; + } + + + } +} diff --git a/crawl/tests/ICMP.cs b/crawl/tests/ICMP.cs new file mode 100644 index 0000000..03870cc --- /dev/null +++ b/crawl/tests/ICMP.cs @@ -0,0 +1,62 @@ +// /** +// * File: ICMP.cs +// * Author: haraldwolff +// * +// * This file and it's content is copyrighted by the Author and / or copyright holder. +// * Any use wihtout proper permission is illegal and may lead to legal actions. +// * +// * +// **/ +using System; +using System.Net.NetworkInformation; +namespace ln.skyscanner.crawl.tests +{ + public static class ICMP + { + public static bool IsReachable(CrawledHost crawledHost) + { + return true; + + //using (Ping ping = new Ping()) + //{ + // HostCrawl.setState("ICMP check"); + + // int nSuccess = 0; + // long roundTripTime = 0; + + // for (int n = 0; n < 10; n++) + // { + // HostCrawl.setState("ICMP check [{0}/10]", n); + + // PingReply pingReply = ping.Send(CrawledHost.PrimaryIP, 500); + // if (pingReply.Status == IPStatus.Success) + // { + // nSuccess++; + // roundTripTime += pingReply.RoundtripTime; + // } + // else if ((n > 3) && (nSuccess == 0)) + // { + // break; + // } + // } + + // if (nSuccess > 0) + // { + // roundTripTime /= nSuccess; + + // CrawledHost.SetHint("ping.success", true); + // CrawledHost.SetHint("ping.rta", (int)roundTripTime); + // CrawledHost.SetHint("ping.out_of_ten", nSuccess); + // } + // else + // { + // CrawledHost.SetHint("ping.success", false); + // CrawledHost.SetHint("ping.rta", null); + // CrawledHost.SetHint("ping.out_of_ten", 0); + // } + // return CrawledHost.GetHint("ping.success"); + //} + } + + } +} diff --git a/crawl/tests/RFC1213.cs b/crawl/tests/RFC1213.cs new file mode 100644 index 0000000..2e35df3 --- /dev/null +++ b/crawl/tests/RFC1213.cs @@ -0,0 +1,39 @@ +// /** +// * File: RFC1213.cs +// * Author: haraldwolff +// * +// * This file and it's content is copyrighted by the Author and / or copyright holder. +// * Any use wihtout proper permission is illegal and may lead to legal actions. +// * +// * +// **/ +using System; +using ln.snmp.endpoint; +using ln.snmp; +using System.Linq; +using ln.types; +namespace ln.skyscanner.crawl.tests +{ + public static class RFC1213 + { + public static void Check(CrawledHost crawledHost) + { + String[] orIDS = crawledHost.GetHint("snmp.orids", new string[0]); + String prID = crawledHost.GetHint("snmp.sysObjectID", ""); + + if (orIDS.Contains("1.3.6.1.2.1.31") || prID.Equals("1.3.6.1.4.1.14988.1")) + { + using (SnmpInterface snmp = SNMP.GetSnmpInterface(crawledHost)) + { + ln.snmp.rfc1213.RFC1213.Interface[] interfaces = ln.snmp.rfc1213.RFC1213.GetInterfaces(snmp); + if (interfaces.Length > 0) + { + crawledHost.Interfaces = interfaces; + crawledHost.SetHint("rfc1213", true); + } + } + } + } + } +} + diff --git a/crawl/tests/SNMP.cs b/crawl/tests/SNMP.cs new file mode 100644 index 0000000..f8d14a9 --- /dev/null +++ b/crawl/tests/SNMP.cs @@ -0,0 +1,244 @@ +// /** +// * File: SNMP.cs +// * Author: haraldwolff +// * +// * This file and it's content is copyrighted by the Author and / or copyright holder. +// * Any use wihtout proper permission is illegal and may lead to legal actions. +// * +// * +// **/ +using System; +using System.Net; +using ln.snmp; +using ln.snmp.endpoint; +using ln.types; +using ln.snmp.types; +using System.Runtime.Remoting.Messaging; +namespace ln.skyscanner.crawl.tests +{ + public static class SNMP + { + static string[] defaultCommunities = new string[] { "VhclfC7lfIojYZ", "Vhclf(C7$lfIojYZ", "ByFR4oW98hap", "qVy3hnZJ2fov" }; + + public static bool HasSNMP(CrawledHost crawledHost) + { + if (!TestCurrentHints(crawledHost)) + { + + } + + return false; + } + + + public static bool TestCurrentHints(CrawledHost crawledHost) + { + SnmpInterface snmpEndpoint = GetSnmpInterface(crawledHost); + if (snmpEndpoint != null) + { + using (snmpEndpoint) + { + try + { + Variable prID = snmpEndpoint.snmpGet("1.3.6.1.2.1.1.2.0"); + } catch (SnmpError) + { + return false; + } catch (TimeoutException) + { + return false; + } + return true; + } + } + return false; + } + + + private static bool TestSnmpV3(CrawledHost crawledHost) + { + CIDR[] ips = crawledHost.IPAddresses; + + foreach (CIDR ip in crawledHost.IPAddresses) + { + using (USMEndpoint v3endpoint = new USMEndpoint(SNMPEngine.DefaultEngine, new IPEndPoint(ip, 161))) + { + try + { + v3endpoint.QueryEngineID(); + } + catch (TimeoutException) + { + } + + if (v3endpoint.RemoteEngineID != null) + { + crawledHost.SetHint("snmp.version", 3); + + bool replied = false; + int c = 0; + + foreach (string community in defaultCommunities) + { + c++; + + v3endpoint.Username = "skytron"; + v3endpoint.AuthMethod = SnmpV3AuthMethod.SHA; + v3endpoint.AuthKeyPhrase = community; + + try + { + Variable prID = v3endpoint.snmpGet("1.3.6.1.2.1.1.2.0"); + + crawledHost.SetHint("snmp.username", "skytron"); + crawledHost.SetHint("snmp.authkey", community); + crawledHost.SetHint("snmp.address", ip); + + return true; + } + catch (TimeoutException) + { + } + } + } + } + } + return false; + } + + private static bool TestSnmpV2(CrawledHost crawledHost) + { + CIDR[] ips = crawledHost.IPAddresses; + + foreach (CIDR ip in crawledHost.IPAddresses) + { + using (SnmpV2Endpoint v2endpoint = new SnmpV2Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(crawledHost.PrimaryIP, 161))) + { + foreach (String community in defaultCommunities) + { + v2endpoint.CommunityString = community; + try + { + Variable prID = v2endpoint.snmpGet("1.3.6.1.2.1.1.2.0"); + + crawledHost.SetHint("snmp.version", 2); + crawledHost.SetHint("snmp.community", community); + crawledHost.SetHint("snmp.address", ip); + + return true; + } + catch (SnmpError) + { + } + catch (TimeoutException) + { + } + } + } + } + return false; + } + + private static bool TestSnmpV1(CrawledHost crawledHost) + { + CIDR[] ips = crawledHost.IPAddresses; + + foreach (CIDR ip in crawledHost.IPAddresses) + { + using (SnmpV1Endpoint v1endpoint = new SnmpV1Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(crawledHost.PrimaryIP, 161))) + { + foreach (String community in defaultCommunities) + { + v1endpoint.CommunityString = community; + try + { + Variable prID = v1endpoint.snmpGet("1.3.6.1.2.1.1.2.0"); + + crawledHost.SetHint("snmp.version", 1); + crawledHost.SetHint("snmp.community", community); + crawledHost.SetHint("snmp.address", ip); + + return true; + } + catch (SnmpError) + { + } + catch (TimeoutException) + { + } + } + } + } + return false; + } + + public static bool TestDefaults(CrawledHost crawledHost) + { + if (TestSnmpV3(crawledHost) ? true : TestSnmpV2(crawledHost) ? true : TestSnmpV1(crawledHost)) + { + using (SnmpInterface snmp = GetSnmpInterface(crawledHost)) + { + Variable prID = snmp.snmpGet("1.3.6.1.2.1.1.2.0"); + crawledHost.SetHint("snmp.sysObjectID", (prID as ObjectIdentifier).AsString); + + try + { + Sequence[] seqORids = snmp.snmpWalk("1.3.6.1.2.1.1.9.1.2").ToArray(); + string[] ORids = new string[seqORids.Length]; + + for (int n = 0; n < ORids.Length; n++) + { + ORids[n] = (seqORids[n].Items[1] as ObjectIdentifier).AsString; + } + + crawledHost.SetHint("snmp.orids", ORids); + } + catch (TimeoutException) + { } + } + return true; + } + else + { + crawledHost.SetHint("snmp.version", null); + crawledHost.SetHint("snmp.username", null); + crawledHost.SetHint("snmp.authkey", null); + crawledHost.SetHint("snmp.community", null); + crawledHost.SetHint("snmp.sysObjectID", null); + + return false; + } + } + + + + public static SnmpInterface GetSnmpInterface(CrawledHost crawledHost) + { + int snmpVersion = crawledHost.GetHint("snmp.version", -1); + CIDR snmpAddress = crawledHost.GetHint("snmp.address", crawledHost.PrimaryIP); + + switch (snmpVersion) + { + case 1: + SnmpV1Endpoint v1 = new SnmpV1Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(snmpAddress, 161)); + v1.CommunityString = crawledHost.GetHint("snmp.community"); + return v1; + case 2: + SnmpV2Endpoint v2 = new SnmpV2Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(snmpAddress, 161)); + v2.CommunityString = crawledHost.GetHint("snmp.community"); + return v2; + case 3: + USMEndpoint endpoint = new USMEndpoint(SNMPEngine.DefaultEngine, new System.Net.IPEndPoint(snmpAddress, 161)); + endpoint.AuthMethod = SnmpV3AuthMethod.SHA; + endpoint.Username = crawledHost.GetHint("snmp.username"); + endpoint.AuthKeyPhrase = crawledHost.GetHint("snmp.authkey"); + return endpoint; + default: + return null; + } + + } + + + } +} diff --git a/crawl/tests/SSH.cs b/crawl/tests/SSH.cs new file mode 100644 index 0000000..b6eaf3a --- /dev/null +++ b/crawl/tests/SSH.cs @@ -0,0 +1,106 @@ +// /** +// * File: SSH.cs +// * Author: haraldwolff +// * +// * This file and it's content is copyrighted by the Author and / or copyright holder. +// * Any use wihtout proper permission is illegal and may lead to legal actions. +// * +// * +// **/ +using System; +using Renci.SshNet; +using ln.types; +using Renci.SshNet.Common; +using System.Net.Sockets; +namespace ln.skyscanner.crawl.tests +{ + public static class SSH + { + static string[] defaultPasswords = new string[] + { + "MNX3oTzhp9am", + "f1whWdj5E2Mo", + "f1whWdj5", + "0Sl71eGw", + "0Sl71eGwVdjI6WeW", + "67E3xpTc", + "67E3xpTcMbwR", + "v1kXbeCux0Td", + "v1kXbeCu", + "YNZRtVUFH94b", + "67E3xpTcMbwR", + "v1kXbeCux0Td", + "DVqxof1JQ9at" + }; + + + public static bool CanConnect(CrawledHost crawledHost) + { + int sshPort = crawledHost.GetHint("ssh.port", -1); + CIDR sshIP = crawledHost.GetHint("ssh.ip", null); + string sshUser = crawledHost.GetHint("ssh.login", null); + string sshPassword = crawledHost.GetHint("ssh.password", null); + + if ((sshPort == -1) || !CanConnect(crawledHost,sshIP.Host.ToString(),sshPort,sshUser,sshPassword)) + { + if (!Scan(crawledHost)) + { + crawledHost.SetHint("ssh.port", -1); + crawledHost.SetHint("ssh.ip", null); + crawledHost.SetHint("ssh.login", null); + crawledHost.SetHint("ssh.password", null); + crawledHost.SetHint("ssh.version", null); + return false; + } + } + return true; + } + + private static bool Scan(CrawledHost crawledHost) + { + foreach (CIDR ip in crawledHost.IPAddresses) + { + foreach (int port in new int[] { 13022, 22 }) + { + foreach (string password in defaultPasswords) + { + if (CanConnect(crawledHost, ip.Host.ToString(), port, "skytron", password)) + return true; + } + } + } + return false; + } + + private static bool CanConnect(CrawledHost crawledHost, String host, int port, string username, string password) + { + using (SshClient client = new SshClient(host, port, username, password)) + { + client.ConnectionInfo.Timeout = TimeSpan.FromSeconds(1); + try + { + client.Connect(); + + crawledHost.SetHint("ssh.port", client.ConnectionInfo.Port); + crawledHost.SetHint("ssh.ip", client.ConnectionInfo.Host); + crawledHost.SetHint("ssh.login", client.ConnectionInfo.Username); + crawledHost.SetHint("ssh.password", password); + crawledHost.SetHint("ssh.version", client.ConnectionInfo.ServerVersion); + + client.Disconnect(); + return true; + } + catch (SshException) + { + } + catch (SocketException) + { + } + } + return false; + } + + + + } +} diff --git a/ln.skyscanner.csproj b/ln.skyscanner.csproj index 3acaab4..a8c400d 100644 --- a/ln.skyscanner.csproj +++ b/ln.skyscanner.csproj @@ -36,6 +36,10 @@ ..\packages\Newtonsoft.Json.12.0.1\lib\net45\Newtonsoft.Json.dll + + ..\packages\Castle.Core.4.3.1\lib\net45\Castle.Core.dll + + @@ -55,8 +59,13 @@ - - + + + + + + + @@ -132,6 +141,7 @@ + diff --git a/packages.config b/packages.config index 78afeed..b189b09 100644 --- a/packages.config +++ b/packages.config @@ -1,5 +1,6 @@  + \ No newline at end of file diff --git a/templates/static/crawlerhosts.html b/templates/static/crawlerhosts.html index b072292..fe4cc22 100644 --- a/templates/static/crawlerhosts.html +++ b/templates/static/crawlerhosts.html @@ -51,6 +51,11 @@