WIP
Harald Wolff
parent
834b2d9c48
commit
f1f325a16a
|
|
@ -0,0 +1,76 @@
|
|||
// /**
|
||||
// * File: Service.cs
|
||||
// * Author: haraldwolff
|
||||
// *
|
||||
// * This file and it's content is copyrighted by the Author and / or copyright holder.
|
||||
// * Any use wihtout proper permission is illegal and may lead to legal actions.
|
||||
// *
|
||||
// *
|
||||
// **/
|
||||
using System;
|
||||
using ln.logging;
|
||||
using System.Linq.Expressions;
|
||||
using ln.perfdb;
|
||||
namespace ln.skyscanner
|
||||
{
|
||||
public abstract class Check
|
||||
{
|
||||
public readonly String CheckID;
|
||||
|
||||
public DateTime LastCheck;
|
||||
public DateTime NextCheck;
|
||||
public TimeSpan CheckInterval;
|
||||
|
||||
public long LastCheckTime;
|
||||
|
||||
public String Name;
|
||||
|
||||
public Check()
|
||||
{
|
||||
CheckInterval = TimeSpan.FromSeconds(60);
|
||||
}
|
||||
|
||||
public Check(String checkID)
|
||||
{
|
||||
CheckID = checkID;
|
||||
CheckInterval = TimeSpan.FromSeconds(60);
|
||||
}
|
||||
|
||||
public string GetPerfName(string perfName)
|
||||
{
|
||||
return string.Format("{0}_{1}",CheckID,perfName);
|
||||
}
|
||||
|
||||
public void Mark()
|
||||
{
|
||||
NextCheck += CheckInterval;
|
||||
if (NextCheck < DateTime.Now)
|
||||
NextCheck = DateTime.Now + CheckInterval;
|
||||
}
|
||||
|
||||
public void Run(IPerfFileProvider perfProvider)
|
||||
{
|
||||
long tStart = DateTimeOffset.Now.ToUnixTimeMilliseconds();
|
||||
|
||||
try
|
||||
{
|
||||
CheckImplementation(perfProvider);
|
||||
|
||||
LastCheck = DateTime.Now;
|
||||
} catch (Exception e)
|
||||
{
|
||||
Logging.Log(LogLevel.WARNING, "Check {0} caught exception: {1}", CheckID, e);
|
||||
Logging.Log(e);
|
||||
}
|
||||
|
||||
LastCheckTime = DateTimeOffset.Now.ToUnixTimeMilliseconds() - tStart;
|
||||
}
|
||||
|
||||
public abstract void CheckImplementation(IPerfFileProvider perfProvider);
|
||||
|
||||
public override int GetHashCode()
|
||||
{
|
||||
return CheckID.GetHashCode();
|
||||
}
|
||||
}
|
||||
}
|
||||
28
Program.cs
28
Program.cs
|
|
@ -25,6 +25,10 @@ using System.Linq;
|
|||
using ln.types;
|
||||
using System.Runtime.InteropServices;
|
||||
using ln.snmp.rfc1213;
|
||||
using ln.types.sync;
|
||||
using ln.skyscanner.crawl;
|
||||
using System.Threading;
|
||||
using System.Net.NetworkInformation;
|
||||
|
||||
namespace ln.skyscanner
|
||||
{
|
||||
|
|
@ -47,13 +51,31 @@ namespace ln.skyscanner
|
|||
|
||||
public static void Main(string[] args)
|
||||
{
|
||||
Logger.ConsoleLogger.MaxLogLevel = LogLevel.DEBUG;
|
||||
|
||||
//CIDR cidr = CIDR.Parse("255.255.255.255/28");
|
||||
//Logging.Log(LogLevel.INFO, "{0} {1}",cidr,cidr.ToNetwork());
|
||||
Crawler crawler = new Crawler();
|
||||
|
||||
Logging.Log(LogLevel.DEBUG, "Find host...");
|
||||
|
||||
//return;
|
||||
crawler.Crawl(IPAddress.Parse("10.10.10.2"));
|
||||
|
||||
int n = 0;
|
||||
|
||||
do
|
||||
{
|
||||
Logging.Log(LogLevel.DEBUG, "QueuedJobs: {0} Pool: {1}", crawler.QueuedJobs, String.Join(", ", crawler.ThreadStates));
|
||||
Thread.Sleep(1000);
|
||||
|
||||
if (crawler.QueuedJobs == 0)
|
||||
n++;
|
||||
else
|
||||
n = 0;
|
||||
|
||||
} while (n < 10);
|
||||
|
||||
crawler.Stop();
|
||||
crawler._CrawlPool.Save();
|
||||
return;
|
||||
|
||||
//PerfFile perfFile = new PerfFile("test.lnpv");
|
||||
//perfFile.Open();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,72 @@
|
|||
// /**
|
||||
// * File: Hostalive.cs
|
||||
// * Author: haraldwolff
|
||||
// *
|
||||
// * This file and it's content is copyrighted by the Author and / or copyright holder.
|
||||
// * Any use wihtout proper permission is illegal and may lead to legal actions.
|
||||
// *
|
||||
// *
|
||||
// **/
|
||||
using System;
|
||||
using ln.types;
|
||||
using ln.skyscanner.entities;
|
||||
using ln.perfdb;
|
||||
using System.Net.NetworkInformation;
|
||||
using ln.logging;
|
||||
using ln.perfdb.storage;
|
||||
namespace ln.skyscanner.check
|
||||
{
|
||||
public class Hostalive : Check
|
||||
{
|
||||
public static String CalcCheckName(Node host)
|
||||
{
|
||||
return String.Format("hostalive-{0}", host.PrimaryIP.ToString());
|
||||
}
|
||||
|
||||
|
||||
public Node Host;
|
||||
|
||||
public Hostalive(Node node)
|
||||
:base(CalcCheckName(node))
|
||||
{
|
||||
Host = node;
|
||||
}
|
||||
|
||||
public override void CheckImplementation(IPerfFileProvider perfProvider)
|
||||
{
|
||||
Ping ping = new Ping();
|
||||
long roundTripTime = 0;
|
||||
int success = 0;
|
||||
int n;
|
||||
|
||||
for (n = 0; n < 4; n++)
|
||||
{
|
||||
PingReply reply = ping.Send(Host.PrimaryIP, 250);
|
||||
if (reply.Status == IPStatus.Success)
|
||||
{
|
||||
success++;
|
||||
roundTripTime += reply.RoundtripTime;
|
||||
}
|
||||
}
|
||||
|
||||
float fSuccess = (float)success / (float)n;
|
||||
|
||||
PerfFile pfSuccess = perfProvider.GetPerfFile(GetPerfName("LOSS"));
|
||||
pfSuccess.Write(new PerfValue(fSuccess));
|
||||
|
||||
if (success > 0)
|
||||
{
|
||||
roundTripTime /= success;
|
||||
Logging.Log(LogLevel.INFO, "HOSTALIVE: IP {0} reachable ({1}/10) {2}ms", Host.PrimaryIP, success, roundTripTime);
|
||||
|
||||
PerfFile pfRoundTrip = perfProvider.GetPerfFile(GetPerfName("RTA"));
|
||||
pfRoundTrip.Write(new PerfValue(roundTripTime));
|
||||
}
|
||||
else
|
||||
{
|
||||
Logging.Log(LogLevel.INFO, "HOSTALIVE: IP {0} unreachable", Host.PrimaryIP);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
// /**
|
||||
// * File: CrawledHost.cs
|
||||
// * Author: haraldwolff
|
||||
// *
|
||||
// * This file and it's content is copyrighted by the Author and / or copyright holder.
|
||||
// * Any use wihtout proper permission is illegal and may lead to legal actions.
|
||||
// *
|
||||
// *
|
||||
// **/
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using ln.types;
|
||||
using ln.types.sync;
|
||||
using System.Net;
|
||||
using System.IO;
|
||||
using System.Net.NetworkInformation;
|
||||
using ln.logging;
|
||||
namespace ln.skyscanner.crawl
|
||||
{
|
||||
public class CrawledHost : Syncable
|
||||
{
|
||||
[Unsynced]
|
||||
private Crawler crawler;
|
||||
public Crawler Crawler { get => crawler; set => crawler = value; }
|
||||
|
||||
private Guid id;
|
||||
public Guid ID { get => id; set => id = value; }
|
||||
|
||||
private List<CIDR> ipaddresses = new List<CIDR>();
|
||||
public List<CIDR> IPAddresses => ipaddresses;
|
||||
|
||||
|
||||
|
||||
public CrawledHost()
|
||||
{ }
|
||||
|
||||
public CrawledHost(Crawler crawler)
|
||||
{
|
||||
Crawler = crawler;
|
||||
ID = Guid.NewGuid();
|
||||
|
||||
FileName = Path.Combine(crawler.BasePath, String.Format("{0}.ch", ID.ToString()));
|
||||
}
|
||||
public CrawledHost(Crawler crawler, Guid id)
|
||||
:this(crawler)
|
||||
{
|
||||
ID = id;
|
||||
FileName = Path.Combine(crawler.BasePath, String.Format("{0}.ch", ID.ToString()));
|
||||
|
||||
Load();
|
||||
}
|
||||
public CrawledHost(Crawler crawler, string filename)
|
||||
: base(filename)
|
||||
{
|
||||
}
|
||||
|
||||
public void Crawl()
|
||||
{
|
||||
Logging.Log(LogLevel.INFO, "Checking {0}", id);
|
||||
|
||||
Ping ping = new Ping();
|
||||
|
||||
foreach (CIDR ip in IPAddresses)
|
||||
{
|
||||
PingReply pingReply = ping.Send(ip);
|
||||
Logging.Log(LogLevel.INFO, "IP {0} Ping: {1}",ip,pingReply.RoundtripTime);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return String.Format("[CrawledHost ID={0} IPAddress=({1})]",ID,String.Join(", ",IPAddresses));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,233 @@
|
|||
// /**
|
||||
// * File: Crawler.cs
|
||||
// * Author: haraldwolff
|
||||
// *
|
||||
// * This file and it's content is copyrighted by the Author and / or copyright holder.
|
||||
// * Any use wihtout proper permission is illegal and may lead to legal actions.
|
||||
// *
|
||||
// *
|
||||
// **/
|
||||
using System;
|
||||
using ln.skyscanner.threads;
|
||||
using System.Net;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using ln.logging;
|
||||
using ln.types;
|
||||
using System.Linq;
|
||||
using ln.types.serialize;
|
||||
using ln.skyscanner.entities;
|
||||
using System.Net.NetworkInformation;
|
||||
using ln.snmp;
|
||||
using ln.snmp.endpoint;
|
||||
using ln.snmp.rfc1213;
|
||||
using ln.perfdb;
|
||||
using ln.perfdb.storage;
|
||||
using ln.skyscanner.check;
|
||||
using System.Threading;
|
||||
|
||||
namespace ln.skyscanner.crawl
|
||||
{
|
||||
public class Crawler : IPerfFileProvider
|
||||
{
|
||||
public String BasePath { get; set; }
|
||||
public String PerfPath => Path.Combine(BasePath, "perfdb");
|
||||
|
||||
bool stopping;
|
||||
|
||||
Pool crawlThreadPool = new Pool();
|
||||
|
||||
public PoolThreadState[] ThreadStates => crawlThreadPool.ThreadStates;
|
||||
public int QueuedJobs => crawlThreadPool.QueuedJobs;
|
||||
|
||||
public DiskObject<CrawlPool> _CrawlPool;
|
||||
public CrawlPool CrawlPool => _CrawlPool.Instance;
|
||||
|
||||
public SNMPEngine SNMPEngine { get; }
|
||||
|
||||
Thread threadChecker;
|
||||
|
||||
public Crawler()
|
||||
{
|
||||
BasePath = Path.GetFullPath("./crawler");
|
||||
|
||||
if (!Directory.Exists(BasePath))
|
||||
Directory.CreateDirectory(BasePath);
|
||||
if (!Directory.Exists(PerfPath))
|
||||
Directory.CreateDirectory(PerfPath);
|
||||
|
||||
|
||||
SNMPEngine = new SNMPEngine();
|
||||
|
||||
_CrawlPool = new DiskObject<CrawlPool>(String.Format("{0}/pool",BasePath));
|
||||
|
||||
crawlThreadPool.NumThreads = 12;
|
||||
|
||||
threadChecker = new Thread(Checker);
|
||||
threadChecker.Start();
|
||||
}
|
||||
|
||||
public void Stop()
|
||||
{
|
||||
stopping = true;
|
||||
crawlThreadPool.Close();
|
||||
SNMPEngine.Close();
|
||||
_CrawlPool.Save();
|
||||
}
|
||||
|
||||
public void Enqueue(JobDelegate job)
|
||||
{
|
||||
crawlThreadPool.Enqueue(job);
|
||||
}
|
||||
|
||||
public void Crawl(IPAddress host)
|
||||
{
|
||||
crawlThreadPool.Enqueue(() => crawlHost(host));
|
||||
}
|
||||
|
||||
private void crawlHost(IPAddress host)
|
||||
{
|
||||
Ping ping = new Ping();
|
||||
PingReply pingReply = ping.Send(host,500);
|
||||
|
||||
if (pingReply.Status != IPStatus.Success)
|
||||
{
|
||||
Logging.Log(LogLevel.INFO, "Host not reachable: {0} {1}", host, pingReply.Status);
|
||||
}
|
||||
else
|
||||
{
|
||||
//SnmpV1Endpoint v1endpoint = new SnmpV1Endpoint(engine, new IPEndPoint(IPAddress.Parse("10.75.1.10"), 161), "ByFR4oW98hap");
|
||||
//SnmpV2Endpoint v2endpoint = new SnmpV2Endpoint(engine, new IPEndPoint(IPAddress.Parse("10.113.254.4"), 161), "ghE7wUmFPoPpkRno");
|
||||
|
||||
USMEndpoint v3endpoint = new USMEndpoint(SNMPEngine, new IPEndPoint(host, 161));
|
||||
v3endpoint.AuthMethod = SnmpV3AuthMethod.SHA;
|
||||
v3endpoint.AuthKeyPhrase = "qVy3hnZJ2fov";
|
||||
v3endpoint.Username = "skytron";
|
||||
|
||||
try
|
||||
{
|
||||
RFC1213.Interface[] interfaces = RFC1213.GetInterfaces(v3endpoint);
|
||||
|
||||
foreach (RFC1213.Interface netIf in interfaces)
|
||||
{
|
||||
Logging.Log(LogLevel.INFO, "Interface: {0}", netIf);
|
||||
|
||||
foreach (CIDR ip in netIf.IPAddresses)
|
||||
{
|
||||
Subnet subnet = CrawlPool.GetSubnet(ip.Network);
|
||||
if ((DateTime.Now - subnet.LastScanned).Hours >= 1)
|
||||
{
|
||||
Enqueue(() => crawlSubnet(ip.Network));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
Logging.Log(LogLevel.INFO, "Host: {0} SNMP communication timed out.", host);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public void crawlSubnet(CIDR subnet)
|
||||
{
|
||||
Ping ping = new Ping();
|
||||
|
||||
Subnet sub = CrawlPool.GetSubnet(subnet);
|
||||
sub.LastScanned = DateTime.Now;
|
||||
|
||||
Logging.Log(LogLevel.INFO, "Scanning {0}", subnet);
|
||||
|
||||
foreach (CIDR ip in subnet)
|
||||
{
|
||||
long roundTripTime = 0;
|
||||
int success = 0;
|
||||
|
||||
for (int n = 0; n < 4; n++)
|
||||
{
|
||||
PingReply reply = ping.Send(ip,250);
|
||||
if (reply.Status == IPStatus.Success)
|
||||
{
|
||||
success++;
|
||||
roundTripTime += reply.RoundtripTime;
|
||||
}
|
||||
}
|
||||
|
||||
if (success > 0)
|
||||
{
|
||||
roundTripTime /= success;
|
||||
Logging.Log(LogLevel.INFO, "IP {0} reachable ({1}/10) {2}ms", ip, success, roundTripTime);
|
||||
|
||||
Node node = CrawlPool.GetNode(ip);
|
||||
if ((DateTime.Now - node.LastSeen ).Hours > 0)
|
||||
{
|
||||
Enqueue(() => crawlHost(ip));
|
||||
}
|
||||
|
||||
string checkID = Hostalive.CalcCheckName(node);
|
||||
if (!CrawlPool.Checks.ContainsKey(checkID))
|
||||
{
|
||||
Hostalive hostalive = new Hostalive(node);
|
||||
CrawlPool.Checks.Add(hostalive.CheckID, hostalive);
|
||||
}
|
||||
|
||||
}
|
||||
else
|
||||
{
|
||||
Logging.Log(LogLevel.INFO, "IP {0} unreachable", ip);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/** Checks **/
|
||||
|
||||
private void Checker()
|
||||
{
|
||||
while (!stopping)
|
||||
{
|
||||
DateTime now = DateTime.Now;
|
||||
|
||||
Check[] checks = CrawlPool.Checks.Values.ToArray();
|
||||
if (checks.Length == 0)
|
||||
Thread.Sleep(1000);
|
||||
|
||||
foreach (Check check in checks)
|
||||
{
|
||||
if (check.NextCheck <= now)
|
||||
{
|
||||
check.Mark();
|
||||
Enqueue(() => check.Run(this));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/** PerfDB **/
|
||||
|
||||
Dictionary<string, PerfFile> perfFiles = new Dictionary<string, PerfFile>();
|
||||
|
||||
public PerfFile GetPerfFile(string name)
|
||||
{
|
||||
if (perfFiles.ContainsKey(name))
|
||||
return perfFiles[name];
|
||||
|
||||
PerfFile perfFile = new PerfFile(Path.Combine(PerfPath, String.Format("{0}.perf", name)));
|
||||
perfFile.Open();
|
||||
|
||||
perfFiles.Add(name, perfFile);
|
||||
|
||||
if (perfFile.FirstSection == null)
|
||||
{
|
||||
PerfFile.PerfFileSection section = new PerfFile.PerfFileSection(perfFile, null, 1440, 60, AggregationMethod.AVERAGE);
|
||||
section = new PerfFile.PerfFileSection(perfFile, section, 1728, 300, AggregationMethod.AVERAGE);
|
||||
section = new PerfFile.PerfFileSection(perfFile, section, 2016, 900, AggregationMethod.AVERAGE);
|
||||
section = new PerfFile.PerfFileSection(perfFile, section, 1344, 3600, AggregationMethod.AVERAGE);
|
||||
section = new PerfFile.PerfFileSection(perfFile, section, 1344, 10800, AggregationMethod.AVERAGE);
|
||||
}
|
||||
return perfFile;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
// /**
|
||||
// * File: CrawlPool.cs
|
||||
// * Author: haraldwolff
|
||||
// *
|
||||
// * This file and it's content is copyrighted by the Author and / or copyright holder.
|
||||
// * Any use wihtout proper permission is illegal and may lead to legal actions.
|
||||
// *
|
||||
// *
|
||||
// **/
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using ln.types;
|
||||
namespace ln.skyscanner.entities
|
||||
{
|
||||
public class CrawlPool
|
||||
{
|
||||
public Dictionary<CIDR, Subnet> Subnets = new Dictionary<CIDR, Subnet>();
|
||||
public Dictionary<CIDR, Node> Nodes = new Dictionary<CIDR, Node>();
|
||||
|
||||
public Dictionary<string, Check> Checks = new Dictionary<String, Check>();
|
||||
|
||||
public CrawlPool()
|
||||
{
|
||||
}
|
||||
|
||||
public Subnet GetSubnet(CIDR network)
|
||||
{
|
||||
if (Subnets.ContainsKey(network))
|
||||
return Subnets[network];
|
||||
|
||||
Subnet subnet = new Subnet(network);
|
||||
Subnets.Add(network,subnet);
|
||||
return subnet;
|
||||
}
|
||||
|
||||
public Node GetNode(CIDR ip)
|
||||
{
|
||||
if (Nodes.ContainsKey(ip))
|
||||
return Nodes[ip];
|
||||
Node node = new Node(ip);
|
||||
Nodes[ip] = node;
|
||||
return node;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -8,23 +8,30 @@
|
|||
// *
|
||||
// **/
|
||||
using System;
|
||||
using ln.types;
|
||||
using System.Collections.Generic;
|
||||
namespace ln.skyscanner.entities
|
||||
{
|
||||
public class Node
|
||||
{
|
||||
public Guid ID { get; private set; }
|
||||
public CIDR PrimaryIP;
|
||||
public String Name;
|
||||
|
||||
public List<NetworkInterface> NetworkInterfaces = new List<NetworkInterface>();
|
||||
|
||||
public DateTime FirstSeen;
|
||||
public DateTime LastSeen;
|
||||
|
||||
public Node()
|
||||
{
|
||||
ID = Guid.NewGuid();
|
||||
FirstSeen = DateTime.Now;
|
||||
}
|
||||
public Node(Guid guid)
|
||||
|
||||
public Node(CIDR primaryIP)
|
||||
{
|
||||
ID = guid;
|
||||
PrimaryIP = primaryIP;
|
||||
FirstSeen = DateTime.Now;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
// /**
|
||||
// * File: NetworkInterface.cs
|
||||
// * File: Router.cs
|
||||
// * Author: haraldwolff
|
||||
// *
|
||||
// * This file and it's content is copyrighted by the Author and / or copyright holder.
|
||||
|
|
@ -8,14 +8,29 @@
|
|||
// *
|
||||
// **/
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using ln.types;
|
||||
namespace ln.skyscanner.entities
|
||||
{
|
||||
public class NetworkInterface
|
||||
{
|
||||
public Node Node { get; private set; }
|
||||
public Node Node;
|
||||
public string Name;
|
||||
public List<CIDR> IPs = new List<CIDR>();
|
||||
|
||||
public NetworkInterface()
|
||||
{ }
|
||||
|
||||
public NetworkInterface(Node node)
|
||||
{
|
||||
Node = node;
|
||||
}
|
||||
public NetworkInterface(Node node,String name)
|
||||
{
|
||||
Node = node;
|
||||
Name = name;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
// /**
|
||||
// * File: Subnet.cs
|
||||
// * Author: haraldwolff
|
||||
// *
|
||||
// * This file and it's content is copyrighted by the Author and / or copyright holder.
|
||||
// * Any use wihtout proper permission is illegal and may lead to legal actions.
|
||||
// *
|
||||
// *
|
||||
// **/
|
||||
using System;
|
||||
using ln.types;
|
||||
namespace ln.skyscanner.entities
|
||||
{
|
||||
public class Subnet
|
||||
{
|
||||
public readonly CIDR Network;
|
||||
public String Name;
|
||||
|
||||
public DateTime FirstSeen;
|
||||
public DateTime LastScanned;
|
||||
|
||||
public Subnet()
|
||||
{
|
||||
FirstSeen = DateTime.Now;
|
||||
}
|
||||
|
||||
public Subnet(CIDR cidr)
|
||||
{
|
||||
Network = cidr;
|
||||
FirstSeen = DateTime.Now;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -40,7 +40,14 @@
|
|||
<Compile Include="identify\NodeIdentifier.cs" />
|
||||
<Compile Include="Target.cs" />
|
||||
<Compile Include="entities\Node.cs" />
|
||||
<Compile Include="entities\NetworkInterface.cs" />
|
||||
<Compile Include="crawl\Crawler.cs" />
|
||||
<Compile Include="threads\Pool.cs" />
|
||||
<Compile Include="crawl\CrawledHost.cs" />
|
||||
<Compile Include="entities\Subnet.cs" />
|
||||
<Compile Include="entities\CrawlPool.cs" />
|
||||
<Compile Include="entities\Router.cs" />
|
||||
<Compile Include="Check.cs" />
|
||||
<Compile Include="check\Hostalive.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="packages.config" />
|
||||
|
|
@ -49,6 +56,8 @@
|
|||
<Folder Include="identify\" />
|
||||
<Folder Include="check\" />
|
||||
<Folder Include="entities\" />
|
||||
<Folder Include="crawl\" />
|
||||
<Folder Include="threads\" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\ln.snmp\ln.snmp.csproj">
|
||||
|
|
|
|||
|
|
@ -0,0 +1,168 @@
|
|||
using System;
|
||||
using System.Threading;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq.Expressions;
|
||||
using ln.logging;
|
||||
using System.Linq;
|
||||
namespace ln.skyscanner.threads
|
||||
{
|
||||
public delegate void JobDelegate();
|
||||
|
||||
public enum PoolThreadState { READY, WORKING, EXITED }
|
||||
|
||||
public class Pool :IDisposable
|
||||
{
|
||||
public PoolThread[] PoolThreads => poolThreads.ToArray();
|
||||
public int NumThreads
|
||||
{
|
||||
get => poolThreads.Count;
|
||||
set => setPoolThreads(value);
|
||||
}
|
||||
|
||||
public PoolThreadState[] ThreadStates => poolThreads.Select((x) => x.State).ToArray();
|
||||
public int QueuedJobs => queuedJobs.Count;
|
||||
|
||||
private List<PoolThread> poolThreads = new List<PoolThread>();
|
||||
private Queue<JobDelegate> queuedJobs = new Queue<JobDelegate>();
|
||||
|
||||
private bool stopping;
|
||||
|
||||
public Pool()
|
||||
{
|
||||
NumThreads = 4;
|
||||
}
|
||||
|
||||
public void Close()
|
||||
{
|
||||
stopping = true;
|
||||
NumThreads = 0;
|
||||
stopping = false;
|
||||
}
|
||||
|
||||
private void setPoolThreads(int count)
|
||||
{
|
||||
lock (poolThreads)
|
||||
{
|
||||
while (poolThreads.Count > count)
|
||||
{
|
||||
poolThreads[poolThreads.Count - 1].Exit();
|
||||
}
|
||||
|
||||
while (poolThreads.Count < count)
|
||||
{
|
||||
poolThreads.Add(new PoolThread(this));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Enqueue(JobDelegate job)
|
||||
{
|
||||
if (stopping)
|
||||
return;
|
||||
|
||||
lock (queuedJobs)
|
||||
{
|
||||
queuedJobs.Enqueue(job);
|
||||
Monitor.Pulse(queuedJobs);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public class PoolThread
|
||||
{
|
||||
public Pool Pool { get; }
|
||||
public Thread Thread { get; }
|
||||
public PoolThreadState State { get; private set; }
|
||||
|
||||
private bool exitCalled { get; set; }
|
||||
|
||||
public PoolThread(Pool pool)
|
||||
{
|
||||
Pool = pool;
|
||||
Thread = new Thread(thread);
|
||||
|
||||
WaitStarted();
|
||||
}
|
||||
|
||||
private void WaitStarted()
|
||||
{
|
||||
lock (this)
|
||||
{
|
||||
Thread.Start();
|
||||
Monitor.Wait(this, 5000);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private void thread()
|
||||
{
|
||||
lock (this)
|
||||
{
|
||||
Monitor.Pulse(this);
|
||||
}
|
||||
|
||||
while (!exitCalled)
|
||||
{
|
||||
try
|
||||
{
|
||||
State = PoolThreadState.WORKING;
|
||||
lock (Pool.queuedJobs)
|
||||
{
|
||||
while (!exitCalled && Pool.queuedJobs.Count > 0)
|
||||
{
|
||||
JobDelegate job = Pool.queuedJobs.Dequeue();
|
||||
Monitor.Exit(Pool.queuedJobs);
|
||||
|
||||
Logging.Log(LogLevel.DEBUG, "PoolThread: starting next job {0}", job);
|
||||
|
||||
try
|
||||
{
|
||||
job();
|
||||
}
|
||||
catch (Exception ie)
|
||||
{
|
||||
Logging.Log(ie);
|
||||
}
|
||||
Monitor.Enter(Pool.queuedJobs);
|
||||
|
||||
State = PoolThreadState.READY;
|
||||
Monitor.Wait(Pool.queuedJobs, 5000);
|
||||
State = PoolThreadState.WORKING;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Logging.Log(e);
|
||||
}
|
||||
}
|
||||
lock (this)
|
||||
{
|
||||
State = PoolThreadState.EXITED;
|
||||
Monitor.Pulse(this);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public void Exit()
|
||||
{
|
||||
lock (this)
|
||||
{
|
||||
exitCalled = true;
|
||||
Monitor.Pulse(this);
|
||||
Monitor.Wait(this);
|
||||
Pool.poolThreads.Remove(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Close();
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue