ln.skyscanner/crawl/Crawler.cs

// /**
//  * File: Crawler.cs
//  * Author: haraldwolff
//  *
//  * This file and it's content is copyrighted by the Author and / or copyright holder.
//  * Any use wihtout proper permission is illegal and may lead to legal actions.
//  *
//  *
//  **/
using System;
using ln.types.threads;
using System.Net;
using System.Collections.Generic;
using System.IO;
using ln.logging;
using ln.types;
using System.Linq;
using ln.types.serialize;
using ln.skyscanner.entities;
using System.Net.NetworkInformation;
using ln.snmp;
using ln.snmp.endpoint;
using ln.snmp.rfc1213;
using ln.perfdb;
using ln.perfdb.storage;
using ln.skyscanner.check;
using System.Threading;
using ln.snmp.types;

namespace ln.skyscanner.crawl
{
    public class Crawler
    {
        public SkyScanner SkyScanner { get; }

        public String BasePath { get; set; }
        public String PerfPath => Path.Combine(BasePath, "perfdb");

        bool stopping;

        Pool crawlThreadPool = new Pool(12);

        public PoolJob[] CurrentJobs => crawlThreadPool.CurrentPoolJobs;
        public PoolJob[] QueuedJobs => crawlThreadPool.QueuedJobs;

        public DiskObject<CrawlPool> _CrawlPool;
        public CrawlPool CrawlPool => _CrawlPool.Instance;

        public SNMPEngine SNMPEngine { get; private set; }

        Thread threadScheduler;

        public Crawler(SkyScanner skyScanner)
        {
            SkyScanner = skyScanner;
            try
            {
                BasePath = Path.Combine(skyScanner.BasePath, "crawler");

                if (!Directory.Exists(BasePath))
                    Directory.CreateDirectory(BasePath);
                if (!Directory.Exists(PerfPath))
                    Directory.CreateDirectory(PerfPath);


                SNMPEngine = new SNMPEngine();
                SNMPEngine.Timeout = 1250;

                _CrawlPool = new DiskObject<CrawlPool>(String.Format("{0}/pool", BasePath));

                threadScheduler = new Thread(scheduler);
                threadScheduler.Start();
            } catch (Exception)
            {
                Stop();

                throw;
            }
        }

        public void Start()
        {
            if (CrawlerState == ComponentState.INITIALIZED)
            {
                stopping = false;
                SNMPEngine = new SNMPEngine();
            }
        }

        public void Stop()
        {
            stopping = true;
            crawlThreadPool.Close();
            SNMPEngine.Close();
            _CrawlPool.Save();
        }

        public ComponentState CrawlerState
        {
            get
            {
                if (stopping)
                    return ComponentState.STOPPING;
                if (crawlThreadPool.CurrentPoolSize == 0)
                    return ComponentState.INITIALIZED;
                return ComponentState.STARTED;
            }
        }


        public void Enqueue(JobDelegate job)
        {
            crawlThreadPool.Enqueue(job);
        }

        public void Crawl(CIDR host)
        {
            crawlThreadPool.Enqueue(() => crawlHost(host.Host));
        }

        private void crawlHost(CIDR host)
        {
            CrawledHost crawledHost = CrawlPool.HostForIP(host);
            crawledHost.LastCheck = DateTime.Now;
            crawledHost.NextCheck = DateTime.Now + TimeSpan.FromHours(1);

            if (crawlPing(crawledHost))
            {
                if (crawledHost.FirstSeen.Equals(DateTime.MinValue))
                    crawledHost.FirstSeen = DateTime.Now;
                crawledHost.LastSeen = DateTime.Now;

                CrawlSNMP(crawledHost);

            }
            else
            {

            }
        }

        public bool crawlPing(CrawledHost crawledHost)
        {
            Ping ping = new Ping();

            int nSuccess = 0;
            long roundTripTime = 0;

            for (int n=0;n<10;n++)
            {
                PingReply pingReply = ping.Send(crawledHost.PrimaryIP, 500);
                if (pingReply.Status == IPStatus.Success)
                {
                    nSuccess++;
                    roundTripTime += pingReply.RoundtripTime;
                }
            }

            if (nSuccess > 0)
            {
                roundTripTime /= nSuccess;

                crawledHost.SetHint("ping.success", true);
                crawledHost.SetHint("ping.rta", (int)roundTripTime);
                crawledHost.SetHint("ping.out_of_ten", nSuccess);
                Logging.Log(LogLevel.INFO, "Host is reachable: {0} RTA={1}ms", crawledHost.PrimaryIP, roundTripTime);
            }
            else
            {
                crawledHost.SetHint("ping.success", false);
                crawledHost.SetHint("ping.rta", null);
                crawledHost.SetHint("ping.out_of_ten", 0);

                Logging.Log(LogLevel.INFO, "Host is unreachable: {0}", crawledHost.PrimaryIP);
            }
            return crawledHost.GetHint<bool>("ping.success");
        }

        public void CrawlSNMP(CrawledHost crawledHost)
        {
            string[] communities = new string[] { "VhclfC7lfIojYZ", "Vhclf(C7$lfIojYZ", "ByFR4oW98hap", "qVy3hnZJ2fov" };

            bool snmpDetected = false;

            using (USMEndpoint v3endpoint = new USMEndpoint(SNMPEngine, new IPEndPoint(crawledHost.PrimaryIP, 161)))
            {

                try
                {
                    v3endpoint.QueryEngineID();
                }
                catch (TimeoutException)
                {
                }

                if (v3endpoint.RemoteEngineID != null)
                {
                    crawledHost.SetHint("snmp.version", 3);
                    Logging.Log(LogLevel.INFO, "{0}: SNMPv3 support detected", crawledHost.PrimaryIP);

                    bool replied = false;

                    foreach (string community in communities)
                    {
                        v3endpoint.Username = "skytron";
                        v3endpoint.AuthMethod = SnmpV3AuthMethod.SHA;
                        v3endpoint.AuthKeyPhrase = community;

                        try
                        {
                            Variable prID = v3endpoint.snmpGet("1.3.6.1.2.1.1.2.0");
                            crawledHost.SetHint("snmp.username", "skytron");
                            crawledHost.SetHint("snmp.authkey", community);
                            crawledHost.SetHint("snmp.sysObjectID", (prID as ObjectIdentifier).AsString);

                            replied = true;
                            break;
                        }
                        catch (TimeoutException)
                        {
                        }
                    }

                    if (replied)
                    {
                        snmpDetected = true;
                    }

                }
            }

            if (!snmpDetected)
            {
                using (SnmpV2Endpoint v2endpoint = new SnmpV2Endpoint(SNMPEngine, new IPEndPoint(crawledHost.PrimaryIP, 161)))
                {
                    foreach (String community in communities)
                    {
                        v2endpoint.CommunityString = community;
                        try
                        {
                            Variable prID = v2endpoint.snmpGet("1.3.6.1.2.1.1.2.0");

                            crawledHost.SetHint("snmp.version", 2);
                            crawledHost.SetHint("snmp.community", community);
                            crawledHost.SetHint("snmp.sysObjectID", (prID as ObjectIdentifier).AsString);

                            snmpDetected = true;
                            break;
                        } catch (TimeoutException)
                        {
                        }

                    }
                }
            }

            if (!snmpDetected)
            {
                using (SnmpV1Endpoint v1endpoint = new SnmpV1Endpoint(SNMPEngine, new IPEndPoint(crawledHost.PrimaryIP, 161)))
                {
                    foreach (String community in communities)
                    {
                        v1endpoint.CommunityString = community;
                        try
                        {
                            Variable prID = v1endpoint.snmpGet("1.3.6.1.2.1.1.2.0");

                            crawledHost.SetHint("snmp.version", 1);
                            crawledHost.SetHint("snmp.community", community);
                            crawledHost.SetHint("snmp.sysObjectID", (prID as ObjectIdentifier).AsString);

                            snmpDetected = true;
                            break;
                        }
                        catch (TimeoutException)
                        {
                        }

                    }
                }
            }

            if (!snmpDetected)
            {
                crawledHost.SetHint("snmp.version", null);
                crawledHost.SetHint("snmp.username", null);
                crawledHost.SetHint("snmp.authkey", null);
                crawledHost.SetHint("snmp.community", null);
                crawledHost.SetHint("snmp.sysObjectID", null);
            }
            else
            {
                try
                {
                    using (SnmpEndpoint endpoint = crawledHost.GetSnmpEndpoint(SNMPEngine))
                    {
                        try
                        {
                            Sequence[] VorIDs = endpoint.snmpWalk("1.3.6.1.2.1.1.9.1.2").ToArray();
                            string[] orids = new string[VorIDs.Length];

                            for (int n = 0; n < orids.Length; n++)
                            {
                                orids[n] = (VorIDs[n].Items[1] as ObjectIdentifier).AsString;
                            }

                            crawledHost.SetHint("snmp.orids", orids);
                        } catch (TimeoutException)
                        { }

                        List<String> ids = new List<string>(crawledHost.GetHint<String[]>("snmp.orids", new string[0]));

                        if (ids.Contains("1.3.6.1.2.1.31") || crawledHost.GetHint<string>("snmp.sysObjectID","").Equals("1.3.6.1.4.1.14988.1"))
                        {
                            CrawlRFC1213(crawledHost, endpoint);
                        }
                    }
                } catch (TimeoutException te)
                {
                    Logging.Log(te);
                }
            }

        }

        private void CrawlRFC1213(CrawledHost crawledHost, SnmpEndpoint endpoint)
        {
            RFC1213.Interface[] interfaces = RFC1213.GetInterfaces(endpoint);

            foreach (RFC1213.Interface netIf in interfaces)
            {
                foreach (CIDR ip in netIf.IPAddresses)
                {
                    if (!crawledHost.IPAddresses.Contains(ip))
                        crawledHost.IPAddresses.Add(ip);

                    CrawlPool.GetSubnet(ip.Network);
                }
            }
        }

        public void CrawlSubnet(CIDR network)
        {
            Ping ping = new Ping();
            Subnet subnet = CrawlPool.GetSubnet(network);
            subnet.LastScan = DateTime.Now;
            subnet.NextScan = DateTime.Now + TimeSpan.FromHours(6);

            Logging.Log(LogLevel.INFO, "Scanning {0}", subnet);

            foreach (CIDR ip in network)
            {
                long roundTripTime = 0;
                int success = 0;

                for (int n = 0; n < 4; n++)
                {
                    PingReply reply = ping.Send(ip,250);
                    if (reply.Status == IPStatus.Success)
                    {
                        success++;
                        roundTripTime += reply.RoundtripTime;
                    }
                }

                if (success > 0)
                {
                    roundTripTime /= success;
                    Logging.Log(LogLevel.INFO, "IP {0} reachable ({1}/10) {2}ms", ip, success, roundTripTime);

                    CrawlPool.HostForIP(ip);
                }
            }
        }



        private void scheduler()
        {
            while (!stopping)
            {
                foreach (CrawledHost crawledHost in CrawlPool.Hosts)
                {
                    if (crawledHost.NextCheck < DateTime.Now)
                    {
                        Crawl(crawledHost.PrimaryIP);
                    }
                }

                foreach (Subnet subnet in CrawlPool.Subnets.ToArray())
                {
                    if (subnet.NextScan < DateTime.Now)
                    {
                        Enqueue(() => CrawlSubnet(subnet.Network));
                    }
                }

                Thread.Sleep(15000);
            }
        }



        ///** PerfDB **/

        //Dictionary<string, PerfFile> perfFiles = new Dictionary<string, PerfFile>();

        //public PerfFile GetPerfFile(string name)
        //{
        //    if (perfFiles.ContainsKey(name))
        //        return perfFiles[name];

        //    PerfFile perfFile = new PerfFile(Path.Combine(PerfPath, String.Format("{0}.perf", name)));
        //    perfFile.Open();

        //    perfFiles.Add(name, perfFile);

        //    if (perfFile.FirstSection == null)
        //    {
        //        PerfFile.PerfFileSection section = new PerfFile.PerfFileSection(perfFile, null, 1440, 60, AggregationMethod.AVERAGE);
        //        section = new PerfFile.PerfFileSection(perfFile, section, 1728, 300, AggregationMethod.AVERAGE);
        //        section = new PerfFile.PerfFileSection(perfFile, section, 2016, 900, AggregationMethod.AVERAGE);
        //        section = new PerfFile.PerfFileSection(perfFile, section, 1344, 3600, AggregationMethod.AVERAGE);
        //        section = new PerfFile.PerfFileSection(perfFile, section, 1344, 10800, AggregationMethod.AVERAGE);
        //    }
        //    return perfFile;
        //}
    }
}



//    //SnmpV1Endpoint v1endpoint = new SnmpV1Endpoint(engine, new IPEndPoint(IPAddress.Parse("10.75.1.10"), 161), "ByFR4oW98hap");
//    //SnmpV2Endpoint v2endpoint = new SnmpV2Endpoint(engine, new IPEndPoint(IPAddress.Parse("10.113.254.4"), 161), "ghE7wUmFPoPpkRno");

//    USMEndpoint v3endpoint = new USMEndpoint(SNMPEngine, new IPEndPoint(host, 161));
//    v3endpoint.AuthMethod = SnmpV3AuthMethod.SHA;
//    v3endpoint.AuthKeyPhrase = "qVy3hnZJ2fov";
//    v3endpoint.Username = "skytron";

//    try
//    {
//        RFC1213.Interface[] interfaces = RFC1213.GetInterfaces(v3endpoint);

//        foreach (RFC1213.Interface netIf in interfaces)
//        {
//            Logging.Log(LogLevel.INFO, "Interface: {0}", netIf);

//            foreach (CIDR ip in netIf.IPAddresses)
//            {
//                Subnet subnet = CrawlPool.GetSubnet(ip.Network);
//                if ((DateTime.Now - subnet.LastScanned).Hours >= 1)
//                {
//                    Enqueue(() => crawlSubnet(ip.Network));
//                }
//            }
//        }
//    }
//    catch (TimeoutException)
//    {
//        Logging.Log(LogLevel.INFO, "Host: {0} SNMP communication timed out.", host);
//    }

//}