263 lines
7.7 KiB
C#
263 lines
7.7 KiB
C#
// /**
|
|
// * File: Crawler.cs
|
|
// * Author: haraldwolff
|
|
// *
|
|
// * This file and it's content is copyrighted by the Author and / or copyright holder.
|
|
// * Any use wihtout proper permission is illegal and may lead to legal actions.
|
|
// *
|
|
// *
|
|
// **/
|
|
using System;
|
|
using ln.types.threads;
|
|
using System.Net;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using ln.logging;
|
|
using ln.types;
|
|
using System.Linq;
|
|
using ln.types.serialize;
|
|
using ln.skyscanner.entities;
|
|
using System.Net.NetworkInformation;
|
|
using ln.snmp;
|
|
using ln.snmp.endpoint;
|
|
using ln.snmp.rfc1213;
|
|
using ln.perfdb;
|
|
using ln.perfdb.storage;
|
|
using ln.skyscanner.check;
|
|
using System.Threading;
|
|
using ln.snmp.types;
|
|
using Newtonsoft.Json;
|
|
using Newtonsoft.Json.Converters;
|
|
|
|
namespace ln.skyscanner.crawl
|
|
{
|
|
public class Crawler
|
|
{
|
|
public SkyScanner SkyScanner { get; }
|
|
|
|
public String BasePath { get; set; }
|
|
public String PerfPath => Path.Combine(BasePath, "perfdb");
|
|
|
|
bool stopping;
|
|
|
|
Pool crawlThreadPool = new Pool(0);
|
|
|
|
public PoolJob[] CurrentJobs => crawlThreadPool.CurrentPoolJobs;
|
|
public PoolJob[] QueuedJobs => crawlThreadPool.QueuedJobs;
|
|
|
|
public DiskObject<CrawlPool> _CrawlPool;
|
|
public CrawlPool CrawlPool => _CrawlPool?.Instance;
|
|
|
|
//public SNMPEngine SNMPEngine { get; private set; }
|
|
|
|
[JsonConverter(typeof(StringEnumConverter))]
|
|
public ComponentState CrawlerState { get; private set; }
|
|
|
|
Thread threadScheduler;
|
|
|
|
public Crawler(SkyScanner skyScanner)
|
|
{
|
|
SkyScanner = skyScanner;
|
|
try
|
|
{
|
|
BasePath = Path.Combine(skyScanner.BasePath, "crawler");
|
|
|
|
if (!Directory.Exists(BasePath))
|
|
Directory.CreateDirectory(BasePath);
|
|
if (!Directory.Exists(PerfPath))
|
|
Directory.CreateDirectory(PerfPath);
|
|
|
|
CrawlerState = ComponentState.INITIALIZED;
|
|
|
|
} catch (Exception)
|
|
{
|
|
Stop();
|
|
|
|
throw;
|
|
}
|
|
}
|
|
|
|
public void Start()
|
|
{
|
|
if (CrawlerState != ComponentState.STOPPING)
|
|
{
|
|
stopping = false;
|
|
|
|
if (_CrawlPool == null)
|
|
_CrawlPool = new DiskObject<CrawlPool>(String.Format("{0}/pool", BasePath));
|
|
|
|
if (crawlThreadPool == null)
|
|
crawlThreadPool = new Pool(128);
|
|
else
|
|
crawlThreadPool.SetPoolSize(128);
|
|
|
|
if (threadScheduler == null)
|
|
{
|
|
threadScheduler = new Thread(scheduler);
|
|
threadScheduler.Start();
|
|
}
|
|
|
|
CrawlerState = ComponentState.STARTED;
|
|
}
|
|
}
|
|
|
|
public void Stop()
|
|
{
|
|
try
|
|
{
|
|
if (CrawlerState == ComponentState.STOPPING)
|
|
{
|
|
if (crawlThreadPool != null)
|
|
{
|
|
crawlThreadPool.Abort();
|
|
|
|
|
|
if (crawlThreadPool.CurrentPoolSize == 0)
|
|
crawlThreadPool = null;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
CrawlerState = ComponentState.STOPPING;
|
|
|
|
stopping = true;
|
|
threadScheduler.Join();
|
|
threadScheduler = null;
|
|
|
|
crawlThreadPool.Close();
|
|
|
|
_CrawlPool.Save();
|
|
|
|
stopping = false;
|
|
CrawlerState = ComponentState.STOPPED;
|
|
}
|
|
} catch (Exception e)
|
|
{
|
|
Logging.Log(e);
|
|
CrawlerState = ComponentState.FAILED;
|
|
}
|
|
}
|
|
|
|
public void Sync()
|
|
{
|
|
lock (CrawlPool)
|
|
{
|
|
_CrawlPool.Save();
|
|
}
|
|
}
|
|
|
|
public void Enqueue(JobDelegate job)
|
|
{
|
|
crawlThreadPool.Enqueue(job);
|
|
}
|
|
|
|
public void Crawl(CIDR cidr)
|
|
{
|
|
try
|
|
{
|
|
if (cidr.MaskWidth == 32)
|
|
{
|
|
CrawledHost crawledHost = CrawlPool.HostForIP(cidr);
|
|
HostCrawl crawlHost = new HostCrawl(this, crawledHost);
|
|
crawlThreadPool.Enqueue(crawlHost);
|
|
}
|
|
else
|
|
{
|
|
Subnet subnet = CrawlPool.GetSubnet(cidr);
|
|
SubnetCrawl crawlSubnet = new SubnetCrawl(this, subnet);
|
|
crawlThreadPool.Enqueue(crawlSubnet);
|
|
}
|
|
} catch (Exception e)
|
|
{
|
|
Logging.Log(e);
|
|
}
|
|
}
|
|
|
|
private void scheduler()
|
|
{
|
|
while (!stopping)
|
|
{
|
|
foreach (CrawledHost crawledHost in CrawlPool.Hosts)
|
|
{
|
|
if (crawledHost.NextCheck < DateTime.Now)
|
|
{
|
|
Crawl(crawledHost.PrimaryIP);
|
|
}
|
|
}
|
|
|
|
foreach (Subnet subnet in CrawlPool.Subnets.ToArray())
|
|
{
|
|
if (subnet.NextScan < (DateTime.Now - TimeSpan.FromDays(1)) && subnet.Network.MaskWidth >= 24)
|
|
{
|
|
Crawl(subnet.Network);
|
|
}
|
|
}
|
|
|
|
Thread.Sleep(2500);
|
|
}
|
|
}
|
|
|
|
|
|
|
|
///** PerfDB **/
|
|
|
|
//Dictionary<string, PerfFile> perfFiles = new Dictionary<string, PerfFile>();
|
|
|
|
//public PerfFile GetPerfFile(string name)
|
|
//{
|
|
// if (perfFiles.ContainsKey(name))
|
|
// return perfFiles[name];
|
|
|
|
// PerfFile perfFile = new PerfFile(Path.Combine(PerfPath, String.Format("{0}.perf", name)));
|
|
// perfFile.Open();
|
|
|
|
// perfFiles.Add(name, perfFile);
|
|
|
|
// if (perfFile.FirstSection == null)
|
|
// {
|
|
// PerfFile.PerfFileSection section = new PerfFile.PerfFileSection(perfFile, null, 1440, 60, AggregationMethod.AVERAGE);
|
|
// section = new PerfFile.PerfFileSection(perfFile, section, 1728, 300, AggregationMethod.AVERAGE);
|
|
// section = new PerfFile.PerfFileSection(perfFile, section, 2016, 900, AggregationMethod.AVERAGE);
|
|
// section = new PerfFile.PerfFileSection(perfFile, section, 1344, 3600, AggregationMethod.AVERAGE);
|
|
// section = new PerfFile.PerfFileSection(perfFile, section, 1344, 10800, AggregationMethod.AVERAGE);
|
|
// }
|
|
// return perfFile;
|
|
//}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// //SnmpV1Endpoint v1endpoint = new SnmpV1Endpoint(engine, new IPEndPoint(IPAddress.Parse("10.75.1.10"), 161), "ByFR4oW98hap");
|
|
// //SnmpV2Endpoint v2endpoint = new SnmpV2Endpoint(engine, new IPEndPoint(IPAddress.Parse("10.113.254.4"), 161), "ghE7wUmFPoPpkRno");
|
|
|
|
// USMEndpoint v3endpoint = new USMEndpoint(SNMPEngine, new IPEndPoint(host, 161));
|
|
// v3endpoint.AuthMethod = SnmpV3AuthMethod.SHA;
|
|
// v3endpoint.AuthKeyPhrase = "qVy3hnZJ2fov";
|
|
// v3endpoint.Username = "skytron";
|
|
|
|
// try
|
|
// {
|
|
// RFC1213.Interface[] interfaces = RFC1213.GetInterfaces(v3endpoint);
|
|
|
|
// foreach (RFC1213.Interface netIf in interfaces)
|
|
// {
|
|
// Logging.Log(LogLevel.INFO, "Interface: {0}", netIf);
|
|
|
|
// foreach (CIDR ip in netIf.IPAddresses)
|
|
// {
|
|
// Subnet subnet = CrawlPool.GetSubnet(ip.Network);
|
|
// if ((DateTime.Now - subnet.LastScanned).Hours >= 1)
|
|
// {
|
|
// Enqueue(() => crawlSubnet(ip.Network));
|
|
// }
|
|
// }
|
|
// }
|
|
// }
|
|
// catch (TimeoutException)
|
|
// {
|
|
// Logging.Log(LogLevel.INFO, "Host: {0} SNMP communication timed out.", host);
|
|
// }
|
|
|
|
//}
|