diff --git a/Check.cs b/Check.cs new file mode 100644 index 0000000..3dffa7e --- /dev/null +++ b/Check.cs @@ -0,0 +1,76 @@ +// /** +// * File: Service.cs +// * Author: haraldwolff +// * +// * This file and it's content is copyrighted by the Author and / or copyright holder. +// * Any use wihtout proper permission is illegal and may lead to legal actions. +// * +// * +// **/ +using System; +using ln.logging; +using System.Linq.Expressions; +using ln.perfdb; +namespace ln.skyscanner +{ + public abstract class Check + { + public readonly String CheckID; + + public DateTime LastCheck; + public DateTime NextCheck; + public TimeSpan CheckInterval; + + public long LastCheckTime; + + public String Name; + + public Check() + { + CheckInterval = TimeSpan.FromSeconds(60); + } + + public Check(String checkID) + { + CheckID = checkID; + CheckInterval = TimeSpan.FromSeconds(60); + } + + public string GetPerfName(string perfName) + { + return string.Format("{0}_{1}",CheckID,perfName); + } + + public void Mark() + { + NextCheck += CheckInterval; + if (NextCheck < DateTime.Now) + NextCheck = DateTime.Now + CheckInterval; + } + + public void Run(IPerfFileProvider perfProvider) + { + long tStart = DateTimeOffset.Now.ToUnixTimeMilliseconds(); + + try + { + CheckImplementation(perfProvider); + + LastCheck = DateTime.Now; + } catch (Exception e) + { + Logging.Log(LogLevel.WARNING, "Check {0} caught exception: {1}", CheckID, e); + Logging.Log(e); + } + + LastCheckTime = DateTimeOffset.Now.ToUnixTimeMilliseconds() - tStart; + } + + public abstract void CheckImplementation(IPerfFileProvider perfProvider); + + public override int GetHashCode() + { + return CheckID.GetHashCode(); + } + } +} diff --git a/Program.cs b/Program.cs index ca44504..b67982d 100644 --- a/Program.cs +++ b/Program.cs @@ -25,6 +25,10 @@ using System.Linq; using ln.types; using System.Runtime.InteropServices; using ln.snmp.rfc1213; +using ln.types.sync; +using ln.skyscanner.crawl; +using System.Threading; +using System.Net.NetworkInformation; namespace ln.skyscanner { @@ -47,13 +51,31 @@ namespace ln.skyscanner public static void Main(string[] args) { + Logger.ConsoleLogger.MaxLogLevel = LogLevel.DEBUG; - //CIDR cidr = CIDR.Parse("255.255.255.255/28"); - //Logging.Log(LogLevel.INFO, "{0} {1}",cidr,cidr.ToNetwork()); + Crawler crawler = new Crawler(); + Logging.Log(LogLevel.DEBUG, "Find host..."); - //return; + crawler.Crawl(IPAddress.Parse("10.10.10.2")); + int n = 0; + + do + { + Logging.Log(LogLevel.DEBUG, "QueuedJobs: {0} Pool: {1}", crawler.QueuedJobs, String.Join(", ", crawler.ThreadStates)); + Thread.Sleep(1000); + + if (crawler.QueuedJobs == 0) + n++; + else + n = 0; + + } while (n < 10); + + crawler.Stop(); + crawler._CrawlPool.Save(); + return; //PerfFile perfFile = new PerfFile("test.lnpv"); //perfFile.Open(); diff --git a/check/Hostalive.cs b/check/Hostalive.cs new file mode 100644 index 0000000..47075c1 --- /dev/null +++ b/check/Hostalive.cs @@ -0,0 +1,72 @@ +// /** +// * File: Hostalive.cs +// * Author: haraldwolff +// * +// * This file and it's content is copyrighted by the Author and / or copyright holder. +// * Any use wihtout proper permission is illegal and may lead to legal actions. +// * +// * +// **/ +using System; +using ln.types; +using ln.skyscanner.entities; +using ln.perfdb; +using System.Net.NetworkInformation; +using ln.logging; +using ln.perfdb.storage; +namespace ln.skyscanner.check +{ + public class Hostalive : Check + { + public static String CalcCheckName(Node host) + { + return String.Format("hostalive-{0}", host.PrimaryIP.ToString()); + } + + + public Node Host; + + public Hostalive(Node node) + :base(CalcCheckName(node)) + { + Host = node; + } + + public override void CheckImplementation(IPerfFileProvider perfProvider) + { + Ping ping = new Ping(); + long roundTripTime = 0; + int success = 0; + int n; + + for (n = 0; n < 4; n++) + { + PingReply reply = ping.Send(Host.PrimaryIP, 250); + if (reply.Status == IPStatus.Success) + { + success++; + roundTripTime += reply.RoundtripTime; + } + } + + float fSuccess = (float)success / (float)n; + + PerfFile pfSuccess = perfProvider.GetPerfFile(GetPerfName("LOSS")); + pfSuccess.Write(new PerfValue(fSuccess)); + + if (success > 0) + { + roundTripTime /= success; + Logging.Log(LogLevel.INFO, "HOSTALIVE: IP {0} reachable ({1}/10) {2}ms", Host.PrimaryIP, success, roundTripTime); + + PerfFile pfRoundTrip = perfProvider.GetPerfFile(GetPerfName("RTA")); + pfRoundTrip.Write(new PerfValue(roundTripTime)); + } + else + { + Logging.Log(LogLevel.INFO, "HOSTALIVE: IP {0} unreachable", Host.PrimaryIP); + } + + } + } +} diff --git a/crawl/CrawledHost.cs b/crawl/CrawledHost.cs new file mode 100644 index 0000000..d48717d --- /dev/null +++ b/crawl/CrawledHost.cs @@ -0,0 +1,77 @@ +// /** +// * File: CrawledHost.cs +// * Author: haraldwolff +// * +// * This file and it's content is copyrighted by the Author and / or copyright holder. +// * Any use wihtout proper permission is illegal and may lead to legal actions. +// * +// * +// **/ +using System; +using System.Collections.Generic; +using System.Security.Cryptography.X509Certificates; +using ln.types; +using ln.types.sync; +using System.Net; +using System.IO; +using System.Net.NetworkInformation; +using ln.logging; +namespace ln.skyscanner.crawl +{ + public class CrawledHost : Syncable + { + [Unsynced] + private Crawler crawler; + public Crawler Crawler { get => crawler; set => crawler = value; } + + private Guid id; + public Guid ID { get => id; set => id = value; } + + private List ipaddresses = new List(); + public List IPAddresses => ipaddresses; + + + + public CrawledHost() + { } + + public CrawledHost(Crawler crawler) + { + Crawler = crawler; + ID = Guid.NewGuid(); + + FileName = Path.Combine(crawler.BasePath, String.Format("{0}.ch", ID.ToString())); + } + public CrawledHost(Crawler crawler, Guid id) + :this(crawler) + { + ID = id; + FileName = Path.Combine(crawler.BasePath, String.Format("{0}.ch", ID.ToString())); + + Load(); + } + public CrawledHost(Crawler crawler, string filename) + : base(filename) + { + } + + public void Crawl() + { + Logging.Log(LogLevel.INFO, "Checking {0}", id); + + Ping ping = new Ping(); + + foreach (CIDR ip in IPAddresses) + { + PingReply pingReply = ping.Send(ip); + Logging.Log(LogLevel.INFO, "IP {0} Ping: {1}",ip,pingReply.RoundtripTime); + } + + } + + public override string ToString() + { + return String.Format("[CrawledHost ID={0} IPAddress=({1})]",ID,String.Join(", ",IPAddresses)); + } + } +} diff --git a/crawl/Crawler.cs b/crawl/Crawler.cs new file mode 100644 index 0000000..b408026 --- /dev/null +++ b/crawl/Crawler.cs @@ -0,0 +1,233 @@ +// /** +// * File: Crawler.cs +// * Author: haraldwolff +// * +// * This file and it's content is copyrighted by the Author and / or copyright holder. +// * Any use wihtout proper permission is illegal and may lead to legal actions. +// * +// * +// **/ +using System; +using ln.skyscanner.threads; +using System.Net; +using System.Collections.Generic; +using System.IO; +using ln.logging; +using ln.types; +using System.Linq; +using ln.types.serialize; +using ln.skyscanner.entities; +using System.Net.NetworkInformation; +using ln.snmp; +using ln.snmp.endpoint; +using ln.snmp.rfc1213; +using ln.perfdb; +using ln.perfdb.storage; +using ln.skyscanner.check; +using System.Threading; + +namespace ln.skyscanner.crawl +{ + public class Crawler : IPerfFileProvider + { + public String BasePath { get; set; } + public String PerfPath => Path.Combine(BasePath, "perfdb"); + + bool stopping; + + Pool crawlThreadPool = new Pool(); + + public PoolThreadState[] ThreadStates => crawlThreadPool.ThreadStates; + public int QueuedJobs => crawlThreadPool.QueuedJobs; + + public DiskObject _CrawlPool; + public CrawlPool CrawlPool => _CrawlPool.Instance; + + public SNMPEngine SNMPEngine { get; } + + Thread threadChecker; + + public Crawler() + { + BasePath = Path.GetFullPath("./crawler"); + + if (!Directory.Exists(BasePath)) + Directory.CreateDirectory(BasePath); + if (!Directory.Exists(PerfPath)) + Directory.CreateDirectory(PerfPath); + + + SNMPEngine = new SNMPEngine(); + + _CrawlPool = new DiskObject(String.Format("{0}/pool",BasePath)); + + crawlThreadPool.NumThreads = 12; + + threadChecker = new Thread(Checker); + threadChecker.Start(); + } + + public void Stop() + { + stopping = true; + crawlThreadPool.Close(); + SNMPEngine.Close(); + _CrawlPool.Save(); + } + + public void Enqueue(JobDelegate job) + { + crawlThreadPool.Enqueue(job); + } + + public void Crawl(IPAddress host) + { + crawlThreadPool.Enqueue(() => crawlHost(host)); + } + + private void crawlHost(IPAddress host) + { + Ping ping = new Ping(); + PingReply pingReply = ping.Send(host,500); + + if (pingReply.Status != IPStatus.Success) + { + Logging.Log(LogLevel.INFO, "Host not reachable: {0} {1}", host, pingReply.Status); + } + else + { + //SnmpV1Endpoint v1endpoint = new SnmpV1Endpoint(engine, new IPEndPoint(IPAddress.Parse("10.75.1.10"), 161), "ByFR4oW98hap"); + //SnmpV2Endpoint v2endpoint = new SnmpV2Endpoint(engine, new IPEndPoint(IPAddress.Parse("10.113.254.4"), 161), "ghE7wUmFPoPpkRno"); + + USMEndpoint v3endpoint = new USMEndpoint(SNMPEngine, new IPEndPoint(host, 161)); + v3endpoint.AuthMethod = SnmpV3AuthMethod.SHA; + v3endpoint.AuthKeyPhrase = "qVy3hnZJ2fov"; + v3endpoint.Username = "skytron"; + + try + { + RFC1213.Interface[] interfaces = RFC1213.GetInterfaces(v3endpoint); + + foreach (RFC1213.Interface netIf in interfaces) + { + Logging.Log(LogLevel.INFO, "Interface: {0}", netIf); + + foreach (CIDR ip in netIf.IPAddresses) + { + Subnet subnet = CrawlPool.GetSubnet(ip.Network); + if ((DateTime.Now - subnet.LastScanned).Hours >= 1) + { + Enqueue(() => crawlSubnet(ip.Network)); + } + } + } + } + catch (TimeoutException) + { + Logging.Log(LogLevel.INFO, "Host: {0} SNMP communication timed out.", host); + } + + } + } + + public void crawlSubnet(CIDR subnet) + { + Ping ping = new Ping(); + + Subnet sub = CrawlPool.GetSubnet(subnet); + sub.LastScanned = DateTime.Now; + + Logging.Log(LogLevel.INFO, "Scanning {0}", subnet); + + foreach (CIDR ip in subnet) + { + long roundTripTime = 0; + int success = 0; + + for (int n = 0; n < 4; n++) + { + PingReply reply = ping.Send(ip,250); + if (reply.Status == IPStatus.Success) + { + success++; + roundTripTime += reply.RoundtripTime; + } + } + + if (success > 0) + { + roundTripTime /= success; + Logging.Log(LogLevel.INFO, "IP {0} reachable ({1}/10) {2}ms", ip, success, roundTripTime); + + Node node = CrawlPool.GetNode(ip); + if ((DateTime.Now - node.LastSeen ).Hours > 0) + { + Enqueue(() => crawlHost(ip)); + } + + string checkID = Hostalive.CalcCheckName(node); + if (!CrawlPool.Checks.ContainsKey(checkID)) + { + Hostalive hostalive = new Hostalive(node); + CrawlPool.Checks.Add(hostalive.CheckID, hostalive); + } + + } + else + { + Logging.Log(LogLevel.INFO, "IP {0} unreachable", ip); + } + } + } + + + /** Checks **/ + + private void Checker() + { + while (!stopping) + { + DateTime now = DateTime.Now; + + Check[] checks = CrawlPool.Checks.Values.ToArray(); + if (checks.Length == 0) + Thread.Sleep(1000); + + foreach (Check check in checks) + { + if (check.NextCheck <= now) + { + check.Mark(); + Enqueue(() => check.Run(this)); + } + } + } + } + + + /** PerfDB **/ + + Dictionary perfFiles = new Dictionary(); + + public PerfFile GetPerfFile(string name) + { + if (perfFiles.ContainsKey(name)) + return perfFiles[name]; + + PerfFile perfFile = new PerfFile(Path.Combine(PerfPath, String.Format("{0}.perf", name))); + perfFile.Open(); + + perfFiles.Add(name, perfFile); + + if (perfFile.FirstSection == null) + { + PerfFile.PerfFileSection section = new PerfFile.PerfFileSection(perfFile, null, 1440, 60, AggregationMethod.AVERAGE); + section = new PerfFile.PerfFileSection(perfFile, section, 1728, 300, AggregationMethod.AVERAGE); + section = new PerfFile.PerfFileSection(perfFile, section, 2016, 900, AggregationMethod.AVERAGE); + section = new PerfFile.PerfFileSection(perfFile, section, 1344, 3600, AggregationMethod.AVERAGE); + section = new PerfFile.PerfFileSection(perfFile, section, 1344, 10800, AggregationMethod.AVERAGE); + } + return perfFile; + } + } +} diff --git a/entities/CrawlPool.cs b/entities/CrawlPool.cs new file mode 100644 index 0000000..249fd87 --- /dev/null +++ b/entities/CrawlPool.cs @@ -0,0 +1,47 @@ +// /** +// * File: CrawlPool.cs +// * Author: haraldwolff +// * +// * This file and it's content is copyrighted by the Author and / or copyright holder. +// * Any use wihtout proper permission is illegal and may lead to legal actions. +// * +// * +// **/ +using System; +using System.Collections.Generic; +using ln.types; +namespace ln.skyscanner.entities +{ + public class CrawlPool + { + public Dictionary Subnets = new Dictionary(); + public Dictionary Nodes = new Dictionary(); + + public Dictionary Checks = new Dictionary(); + + public CrawlPool() + { + } + + public Subnet GetSubnet(CIDR network) + { + if (Subnets.ContainsKey(network)) + return Subnets[network]; + + Subnet subnet = new Subnet(network); + Subnets.Add(network,subnet); + return subnet; + } + + public Node GetNode(CIDR ip) + { + if (Nodes.ContainsKey(ip)) + return Nodes[ip]; + Node node = new Node(ip); + Nodes[ip] = node; + return node; + } + + + } +} diff --git a/entities/Node.cs b/entities/Node.cs index 4c64f79..e926c63 100644 --- a/entities/Node.cs +++ b/entities/Node.cs @@ -8,23 +8,30 @@ // * // **/ using System; +using ln.types; +using System.Collections.Generic; namespace ln.skyscanner.entities { public class Node { - public Guid ID { get; private set; } + public CIDR PrimaryIP; + public String Name; + + public List NetworkInterfaces = new List(); + + public DateTime FirstSeen; + public DateTime LastSeen; public Node() { - ID = Guid.NewGuid(); + FirstSeen = DateTime.Now; } - public Node(Guid guid) + + public Node(CIDR primaryIP) { - ID = guid; + PrimaryIP = primaryIP; + FirstSeen = DateTime.Now; } - - - } } diff --git a/entities/NetworkInterface.cs b/entities/Router.cs similarity index 54% rename from entities/NetworkInterface.cs rename to entities/Router.cs index 74fff26..dc1885f 100644 --- a/entities/NetworkInterface.cs +++ b/entities/Router.cs @@ -1,5 +1,5 @@ // /** -// * File: NetworkInterface.cs +// * File: Router.cs // * Author: haraldwolff // * // * This file and it's content is copyrighted by the Author and / or copyright holder. @@ -8,14 +8,29 @@ // * // **/ using System; +using System.Collections.Generic; +using ln.types; namespace ln.skyscanner.entities { public class NetworkInterface { - public Node Node { get; private set; } + public Node Node; + public string Name; + public List IPs = new List(); + + public NetworkInterface() + { } + public NetworkInterface(Node node) { Node = node; } + public NetworkInterface(Node node,String name) + { + Node = node; + Name = name; + } + } + } diff --git a/entities/Subnet.cs b/entities/Subnet.cs new file mode 100644 index 0000000..2973bbc --- /dev/null +++ b/entities/Subnet.cs @@ -0,0 +1,33 @@ +// /** +// * File: Subnet.cs +// * Author: haraldwolff +// * +// * This file and it's content is copyrighted by the Author and / or copyright holder. +// * Any use wihtout proper permission is illegal and may lead to legal actions. +// * +// * +// **/ +using System; +using ln.types; +namespace ln.skyscanner.entities +{ + public class Subnet + { + public readonly CIDR Network; + public String Name; + + public DateTime FirstSeen; + public DateTime LastScanned; + + public Subnet() + { + FirstSeen = DateTime.Now; + } + + public Subnet(CIDR cidr) + { + Network = cidr; + FirstSeen = DateTime.Now; + } + } +} diff --git a/ln.skyscanner.csproj b/ln.skyscanner.csproj index 7809d13..371ad56 100644 --- a/ln.skyscanner.csproj +++ b/ln.skyscanner.csproj @@ -40,7 +40,14 @@ - + + + + + + + + @@ -49,6 +56,8 @@ + + diff --git a/threads/Pool.cs b/threads/Pool.cs new file mode 100644 index 0000000..89c7175 --- /dev/null +++ b/threads/Pool.cs @@ -0,0 +1,168 @@ +using System; +using System.Threading; +using System.Collections.Generic; +using System.Linq.Expressions; +using ln.logging; +using System.Linq; +namespace ln.skyscanner.threads +{ + public delegate void JobDelegate(); + + public enum PoolThreadState { READY, WORKING, EXITED } + + public class Pool :IDisposable + { + public PoolThread[] PoolThreads => poolThreads.ToArray(); + public int NumThreads + { + get => poolThreads.Count; + set => setPoolThreads(value); + } + + public PoolThreadState[] ThreadStates => poolThreads.Select((x) => x.State).ToArray(); + public int QueuedJobs => queuedJobs.Count; + + private List poolThreads = new List(); + private Queue queuedJobs = new Queue(); + + private bool stopping; + + public Pool() + { + NumThreads = 4; + } + + public void Close() + { + stopping = true; + NumThreads = 0; + stopping = false; + } + + private void setPoolThreads(int count) + { + lock (poolThreads) + { + while (poolThreads.Count > count) + { + poolThreads[poolThreads.Count - 1].Exit(); + } + + while (poolThreads.Count < count) + { + poolThreads.Add(new PoolThread(this)); + } + } + } + + public void Enqueue(JobDelegate job) + { + if (stopping) + return; + + lock (queuedJobs) + { + queuedJobs.Enqueue(job); + Monitor.Pulse(queuedJobs); + } + } + + + + + + public class PoolThread + { + public Pool Pool { get; } + public Thread Thread { get; } + public PoolThreadState State { get; private set; } + + private bool exitCalled { get; set; } + + public PoolThread(Pool pool) + { + Pool = pool; + Thread = new Thread(thread); + + WaitStarted(); + } + + private void WaitStarted() + { + lock (this) + { + Thread.Start(); + Monitor.Wait(this, 5000); + } + + } + + private void thread() + { + lock (this) + { + Monitor.Pulse(this); + } + + while (!exitCalled) + { + try + { + State = PoolThreadState.WORKING; + lock (Pool.queuedJobs) + { + while (!exitCalled && Pool.queuedJobs.Count > 0) + { + JobDelegate job = Pool.queuedJobs.Dequeue(); + Monitor.Exit(Pool.queuedJobs); + + Logging.Log(LogLevel.DEBUG, "PoolThread: starting next job {0}", job); + + try + { + job(); + } + catch (Exception ie) + { + Logging.Log(ie); + } + Monitor.Enter(Pool.queuedJobs); + + State = PoolThreadState.READY; + Monitor.Wait(Pool.queuedJobs, 5000); + State = PoolThreadState.WORKING; + } + } + } + catch (Exception e) + { + Logging.Log(e); + } + } + lock (this) + { + State = PoolThreadState.EXITED; + Monitor.Pulse(this); + } + + } + + public void Exit() + { + lock (this) + { + exitCalled = true; + Monitor.Pulse(this); + Monitor.Wait(this); + Pool.poolThreads.Remove(this); + } + } + } + + + public void Dispose() + { + Close(); + } + } +}