ln.skyscanner/crawl/Crawler.cs

// /**
//  * File: Crawler.cs
//  * Author: haraldwolff
//  *
//  * This file and it's content is copyrighted by the Author and / or copyright holder.
//  * Any use wihtout proper permission is illegal and may lead to legal actions.
//  *
//  *
//  **/
using System;
using ln.types.threads;
using System.Net;
using System.Collections.Generic;
using System.IO;
using ln.logging;
using ln.types;
using System.Linq;
using ln.types.serialize;
using ln.skyscanner.entities;
using System.Net.NetworkInformation;
using ln.snmp;
using ln.snmp.endpoint;
using ln.snmp.rfc1213;
using ln.perfdb;
using ln.perfdb.storage;
using ln.skyscanner.check;
using System.Threading;
using ln.snmp.types;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using ln.types.odb;

namespace ln.skyscanner.crawl
{
    public class Crawler
    {
        public SkyScanner SkyScanner { get; }

        public String BasePath { get; set; }
        public String PerfPath => Path.Combine(BasePath, "perfdb");
        public string PoolPath => Path.Combine(BasePath, "pool");

        bool stopping;

        Pool crawlThreadPool = new Pool(0);

        public PoolJob[] CurrentJobs => crawlThreadPool.CurrentPoolJobs;
        public PoolJob[] QueuedJobs => crawlThreadPool.QueuedJobs;

        public CrawlPool CrawlPool => dbCrawlPool?.Root;
        public ODB<CrawlPool> DBCrawlPool => dbCrawlPool;



        [JsonConverter(typeof(StringEnumConverter))]
        public ComponentState CrawlerState { get; private set; }

        Thread threadScheduler;

        private ODB<CrawlPool> dbCrawlPool;

        public Crawler(SkyScanner skyScanner)
        {
            SkyScanner = skyScanner;
            try
            {
                BasePath = Path.Combine(skyScanner.BasePath, "crawler");

                if (!Directory.Exists(BasePath))
                    Directory.CreateDirectory(BasePath);
                if (!Directory.Exists(PerfPath))
                    Directory.CreateDirectory(PerfPath);

                CrawlerState = ComponentState.INITIALIZED;

            } catch (Exception)
            {
                Stop();

                throw;
            }
        }

        public void Start()
        {
            if (CrawlerState != ComponentState.STOPPING)
            {
                stopping = false;

                dbCrawlPool = new ODB<CrawlPool>(PoolPath);

                if (crawlThreadPool == null)
                    crawlThreadPool = new Pool(128);
                else
                    crawlThreadPool.SetPoolSize(128);

                if (threadScheduler == null)
                {
                    threadScheduler = new Thread(scheduler);
                    threadScheduler.Start();
                }

                CrawlerState = ComponentState.STARTED;
            }
        }

        public void Stop()
        {
            try
            {
                if (CrawlerState == ComponentState.STOPPING)
                {
                    if (crawlThreadPool != null)
                    {
                        crawlThreadPool.Abort();


                        if (crawlThreadPool.CurrentPoolSize == 0)
                            crawlThreadPool = null;
                    }
                }
                else
                {
                    CrawlerState = ComponentState.STOPPING;

                    stopping = true;
                    threadScheduler.Join();
                    threadScheduler = null;

                    crawlThreadPool.Close();

                    Sync();

                    stopping = false;
                    CrawlerState = ComponentState.STOPPED;
                }
            } catch (Exception e)
            {
                Logging.Log(e);
                CrawlerState = ComponentState.FAILED;
            }
        }

        public void Sync()
        {
            Sync(CrawlPool);
        }
        public void Sync(IPersistent persistent)
        {
            lock (CrawlPool)
            {
                dbCrawlPool.SavePersistent(persistent);
            }
        }

        public void Enqueue(JobDelegate job)
        {
            crawlThreadPool.Enqueue(job);
        }

        public void Crawl(CIDR cidr)
        {
            try
            {
                if (cidr.MaskWidth == 32)
                {
                    CrawledHost crawledHost = CrawlPool.HostForIP(cidr);
                    HostCrawl crawlHost = new HostCrawl(this, crawledHost);
                    crawlThreadPool.Enqueue(crawlHost);
                }
                else
                {
                    CrawledSubnet subnet = CrawlPool.GetSubnet(cidr);
                    SubnetCrawl crawlSubnet = new SubnetCrawl(this, subnet);
                    crawlThreadPool.Enqueue(crawlSubnet);
                }
            } catch (Exception e)
            {
                Logging.Log(e);
            }
        }

        private void scheduler()
        {
            while (!stopping)
            {
                foreach (CrawledHost crawledHost in CrawlPool.Hosts)
                {
                    CrawledHost crawledHost2 = CrawlPool.HostForIP(crawledHost.PrimaryIP);

                    if (crawledHost != crawledHost2)
                    {
                        CrawlPool.RemoveHost(crawledHost);
                    } else if (crawledHost.NextCheck < DateTime.Now)
                    {
                        Crawl(crawledHost.PrimaryIP);
                    }
                }

                foreach (CrawledSubnet subnet in CrawlPool.Subnets.ToArray())
                {
                    if (subnet.NextScan < (DateTime.Now - TimeSpan.FromDays(1)) && subnet.Network.MaskWidth >= 24)
                    {
                        Crawl(subnet.Network);
                    }
                }

                Thread.Sleep(2500);
            }
        }



        ///** PerfDB **/

        //Dictionary<string, PerfFile> perfFiles = new Dictionary<string, PerfFile>();

        //public PerfFile GetPerfFile(string name)
        //{
        //    if (perfFiles.ContainsKey(name))
        //        return perfFiles[name];

        //    PerfFile perfFile = new PerfFile(Path.Combine(PerfPath, String.Format("{0}.perf", name)));
        //    perfFile.Open();

        //    perfFiles.Add(name, perfFile);

        //    if (perfFile.FirstSection == null)
        //    {
        //        PerfFile.PerfFileSection section = new PerfFile.PerfFileSection(perfFile, null, 1440, 60, AggregationMethod.AVERAGE);
        //        section = new PerfFile.PerfFileSection(perfFile, section, 1728, 300, AggregationMethod.AVERAGE);
        //        section = new PerfFile.PerfFileSection(perfFile, section, 2016, 900, AggregationMethod.AVERAGE);
        //        section = new PerfFile.PerfFileSection(perfFile, section, 1344, 3600, AggregationMethod.AVERAGE);
        //        section = new PerfFile.PerfFileSection(perfFile, section, 1344, 10800, AggregationMethod.AVERAGE);
        //    }
        //    return perfFile;
        //}
    }
}



//    //SnmpV1Endpoint v1endpoint = new SnmpV1Endpoint(engine, new IPEndPoint(IPAddress.Parse("10.75.1.10"), 161), "ByFR4oW98hap");
//    //SnmpV2Endpoint v2endpoint = new SnmpV2Endpoint(engine, new IPEndPoint(IPAddress.Parse("10.113.254.4"), 161), "ghE7wUmFPoPpkRno");

//    USMEndpoint v3endpoint = new USMEndpoint(SNMPEngine, new IPEndPoint(host, 161));
//    v3endpoint.AuthMethod = SnmpV3AuthMethod.SHA;
//    v3endpoint.AuthKeyPhrase = "qVy3hnZJ2fov";
//    v3endpoint.Username = "skytron";

//    try
//    {
//        RFC1213.Interface[] interfaces = RFC1213.GetInterfaces(v3endpoint);

//        foreach (RFC1213.Interface netIf in interfaces)
//        {
//            Logging.Log(LogLevel.INFO, "Interface: {0}", netIf);

//            foreach (CIDR ip in netIf.IPAddresses)
//            {
//                Subnet subnet = CrawlPool.GetSubnet(ip.Network);
//                if ((DateTime.Now - subnet.LastScanned).Hours >= 1)
//                {
//                    Enqueue(() => crawlSubnet(ip.Network));
//                }
//            }
//        }
//    }
//    catch (TimeoutException)
//    {
//        Logging.Log(LogLevel.INFO, "Host: {0} SNMP communication timed out.", host);
//    }

//}