Skytron
/
ln.skyscanner
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

WIP

broken
Harald Wolff 2019-03-21 13:06:36 +00:00
parent e124f5e885
commit 081d118ced
17 changed files with 261 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Program.cs
View File

@ -31,6 +31,7 @@ using System.Threading;
using System.Net.NetworkInformation;
using ln.skyscanner.crawl.tests;
using Castle.DynamicProxy;
using ln.types.odb;
namespace ln.skyscanner
{
@ -55,6 +56,12 @@ namespace ln.skyscanner
private static void Initialize()
{
//ODB<CrawlPool> dB = new ODB<CrawlPool>("testdb");
//dB.Root.HostForIP(CIDR.Parse("1.2.3.4"));
//dB.Root.HostForIP(CIDR.Parse("2.3.4.5"));
//dB.SavePersistent(dB.RootObject);
}
}

18
crawl/CrawlPool.cs
View File

@ -13,12 +13,13 @@ using ln.types;
using System.Net;
using System.Linq;
using ln.skyscanner.entities;
using ln.types.odb;
namespace ln.skyscanner.crawl
{
public class CrawlPool
public class CrawlPool : Persistent
{
private List<CrawledHost> hosts = new List<CrawledHost>();
private List<Subnet> subnets = new List<Subnet>();
private List<CrawledSubnet> subnets = new List<CrawledSubnet>();
public CrawlPool()
{
@ -30,7 +31,7 @@ namespace ln.skyscanner.crawl
}
}
public Subnet[] Subnets
public CrawledSubnet[] Subnets
{
get
{
@ -62,17 +63,22 @@ namespace ln.skyscanner.crawl
}
}
public Subnet GetSubnet(CIDR network)
public void RemoveHost(CrawledHost crawledHost)
{
hosts.Remove(crawledHost);
}
public CrawledSubnet GetSubnet(CIDR network)
{
lock (this)
{
foreach (Subnet subnet in subnets)
foreach (CrawledSubnet subnet in subnets)
{
if (subnet.Network.Equals(network))
return subnet;
}
Subnet net = new Subnet(network);
CrawledSubnet net = new CrawledSubnet(network);
subnets.Add(net);
return net;
}

3
crawl/CrawledHost.cs
View File

@ -20,10 +20,11 @@ using System.Linq;
using ln.snmp;
using ln.snmp.endpoint;
using ln.snmp.rfc1213;
using ln.types.odb;
namespace ln.skyscanner.crawl
{
public class CrawledHost
public class CrawledHost : Persistent
{
private List<RFC1213.Interface> interfaces = new List<RFC1213.Interface>();
public RFC1213.Interface[] Interfaces

47
crawl/CrawledSubnet.cs 100644
View File

@ -0,0 +1,47 @@
// /**
// * File: Subnet.cs
// * Author: haraldwolff
// *
// * This file and it's content is copyrighted by the Author and / or copyright holder.
// * Any use wihtout proper permission is illegal and may lead to legal actions.
// *
// *
// **/
using System;
using ln.types;
using ln.types.odb;
using System.Collections.Generic;
using System.Linq;
namespace ln.skyscanner.entities
{
public class CrawledSubnet : Persistent
{
public readonly CIDR Network;
public String Name;
public DateTime FirstSeen;
public DateTime LastScan;
public DateTime NextScan;
private CrawledSubnet()
{
FirstSeen = DateTime.Now;
}
public CrawledSubnet(CIDR cidr)
{
Network = cidr;
Name = Network.ToString();
FirstSeen = DateTime.Now;
}
public override string ToString()
{
return Network.ToString();
}
}
}

32
crawl/Crawler.cs
View File

@ -28,6 +28,7 @@ using System.Threading;
using ln.snmp.types;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using ln.types.odb;
namespace ln.skyscanner.crawl
{
@ -37,6 +38,7 @@ namespace ln.skyscanner.crawl
public String BasePath { get; set; }
public String PerfPath => Path.Combine(BasePath, "perfdb");
public string PoolPath => Path.Combine(BasePath, "pool");
bool stopping;
@ -45,16 +47,18 @@ namespace ln.skyscanner.crawl
public PoolJob[] CurrentJobs => crawlThreadPool.CurrentPoolJobs;
public PoolJob[] QueuedJobs => crawlThreadPool.QueuedJobs;
public DiskObject<CrawlPool> _CrawlPool;
public CrawlPool CrawlPool => _CrawlPool?.Instance;
public CrawlPool CrawlPool => dbCrawlPool?.Root;
public ODB<CrawlPool> DBCrawlPool => dbCrawlPool;
//public SNMPEngine SNMPEngine { get; private set; }
[JsonConverter(typeof(StringEnumConverter))]
public ComponentState CrawlerState { get; private set; }
Thread threadScheduler;
private ODB<CrawlPool> dbCrawlPool;
public Crawler(SkyScanner skyScanner)
{
SkyScanner = skyScanner;
@ -83,8 +87,7 @@ namespace ln.skyscanner.crawl
{
stopping = false;
if (_CrawlPool == null)
_CrawlPool = new DiskObject<CrawlPool>(String.Format("{0}/pool", BasePath));
dbCrawlPool = new ODB<CrawlPool>(PoolPath);
if (crawlThreadPool == null)
crawlThreadPool = new Pool(128);
@ -126,7 +129,7 @@ namespace ln.skyscanner.crawl
crawlThreadPool.Close();
_CrawlPool.Save();
Sync();
stopping = false;
CrawlerState = ComponentState.STOPPED;
@ -139,10 +142,14 @@ namespace ln.skyscanner.crawl
}
public void Sync()
{
Sync(CrawlPool);
}
public void Sync(IPersistent persistent)
{
lock (CrawlPool)
{
_CrawlPool.Save();
dbCrawlPool.SavePersistent(persistent);
}
}
@ -163,7 +170,7 @@ namespace ln.skyscanner.crawl
}
else
{
Subnet subnet = CrawlPool.GetSubnet(cidr);
CrawledSubnet subnet = CrawlPool.GetSubnet(cidr);
SubnetCrawl crawlSubnet = new SubnetCrawl(this, subnet);
crawlThreadPool.Enqueue(crawlSubnet);
}
@ -179,13 +186,18 @@ namespace ln.skyscanner.crawl
{
foreach (CrawledHost crawledHost in CrawlPool.Hosts)
{
if (crawledHost.NextCheck < DateTime.Now)
CrawledHost crawledHost2 = CrawlPool.HostForIP(crawledHost.PrimaryIP);
if (crawledHost != crawledHost2)
{
CrawlPool.RemoveHost(crawledHost);
} else if (crawledHost.NextCheck < DateTime.Now)
{
Crawl(crawledHost.PrimaryIP);
}
}
foreach (Subnet subnet in CrawlPool.Subnets.ToArray())
foreach (CrawledSubnet subnet in CrawlPool.Subnets.ToArray())
{
if (subnet.NextScan < (DateTime.Now - TimeSpan.FromDays(1)) && subnet.Network.MaskWidth >= 24)
{

2
crawl/HostCrawl.cs
View File

@ -55,6 +55,8 @@ namespace ln.skyscanner.crawl
CrawledHost.LastCheckTime = DateTime.Now - dateTime;
CrawledHost.LastCheck = dateTime;
CrawledHost.NextCheck = dateTime + TimeSpan.FromHours(1);
Crawler.Sync(CrawledHost);
}
public override int GetHashCode()

18
crawl/SubnetCrawl.cs
View File

@ -15,18 +15,20 @@ using System.Net.NetworkInformation;
using ln.logging;
using Newtonsoft.Json;
using System.Linq;
using ln.types.odb;
namespace ln.skyscanner.crawl
{
public class SubnetCrawl : PoolJob
public class SubnetCrawl : PoolJob, IPersistent
{
[JsonIgnoreAttribute]
public Crawler Crawler { get; }
public Subnet Subnet { get; }
public CrawledSubnet Subnet { get; }
public CIDR Network { get; }
public SubnetCrawl(Crawler crawler, Subnet subnet)
public SubnetCrawl(Crawler crawler, CrawledSubnet subnet)
{
Crawler = crawler;
Subnet = subnet;
@ -34,7 +36,7 @@ namespace ln.skyscanner.crawl
Name = String.Format("Subnet crawl {0}", subnet.Network);
}
public SubnetCrawl(Crawler crawler, Subnet subnet,CIDR network)
public SubnetCrawl(Crawler crawler, CrawledSubnet subnet,CIDR network)
{
Crawler = crawler;
Subnet = subnet;
@ -99,6 +101,9 @@ namespace ln.skyscanner.crawl
Subnet.LastScan = startTime;
Subnet.NextScan = startTime + TimeSpan.FromHours(24);
}
Crawler.Sync(Subnet);
}
public override int GetHashCode()
@ -116,5 +121,10 @@ namespace ln.skyscanner.crawl
}
private Guid PersistenceID = Guid.NewGuid();
public Guid GetPersistenceID()
{
return PersistenceID;
}
}
}

16
crawl/tests/SNMP.cs
View File

@ -24,7 +24,7 @@ namespace ln.skyscanner.crawl.tests
{
if (!TestCurrentHints(crawledHost))
{
return TestDefaults(crawledHost);
}
return false;
@ -228,11 +228,15 @@ namespace ln.skyscanner.crawl.tests
v2.CommunityString = crawledHost.GetHint<string>("snmp.community");
return v2;
case 3:
USMEndpoint endpoint = new USMEndpoint(SNMPEngine.DefaultEngine, new System.Net.IPEndPoint(snmpAddress, 161));
endpoint.AuthMethod = SnmpV3AuthMethod.SHA;
endpoint.Username = crawledHost.GetHint<string>("snmp.username");
endpoint.AuthKeyPhrase = crawledHost.GetHint<string>("snmp.authkey");
return endpoint;
if (crawledHost.GetHint<string>("snmp.username", null) != null)
{
USMEndpoint endpoint = new USMEndpoint(SNMPEngine.DefaultEngine, new System.Net.IPEndPoint(snmpAddress, 161));
endpoint.AuthMethod = SnmpV3AuthMethod.SHA;
endpoint.Username = crawledHost.GetHint<string>("snmp.username");
endpoint.AuthKeyPhrase = crawledHost.GetHint<string>("snmp.authkey");
return endpoint;
}
return null;
default:
return null;
}

11
crawl/tests/SSH.cs
View File

@ -41,7 +41,7 @@ namespace ln.skyscanner.crawl.tests
string sshUser = crawledHost.GetHint<string>("ssh.login", null);
string sshPassword = crawledHost.GetHint<string>("ssh.password", null);
if ((sshPort == -1) || !CanConnect(crawledHost,sshIP.Host.ToString(),sshPort,sshUser,sshPassword))
if ((sshPort == -1) || !CanConnect(crawledHost,sshIP.Host,sshPort,sshUser,sshPassword))
{
if (!Scan(crawledHost))
{
@ -64,7 +64,7 @@ namespace ln.skyscanner.crawl.tests
{
foreach (string password in defaultPasswords)
{
if (CanConnect(crawledHost, ip.Host.ToString(), port, "skytron", password))
if (CanConnect(crawledHost, ip.Host, port, "skytron", password))
return true;
}
}
@ -72,9 +72,9 @@ namespace ln.skyscanner.crawl.tests
return false;
}
private static bool CanConnect(CrawledHost crawledHost, String host, int port, string username, string password)
private static bool CanConnect(CrawledHost crawledHost, CIDR host, int port, string username, string password)
{
using (SshClient client = new SshClient(host, port, username, password))
using (SshClient client = new SshClient(host.ToString(), port, username, password))
{
client.ConnectionInfo.Timeout = TimeSpan.FromSeconds(1);
try
@ -82,7 +82,7 @@ namespace ln.skyscanner.crawl.tests
client.Connect();
crawledHost.SetHint("ssh.port", client.ConnectionInfo.Port);
crawledHost.SetHint("ssh.ip", client.ConnectionInfo.Host);
crawledHost.SetHint("ssh.ip", host);
crawledHost.SetHint("ssh.login", client.ConnectionInfo.Username);
crawledHost.SetHint("ssh.password", password);
crawledHost.SetHint("ssh.version", client.ConnectionInfo.ServerVersion);
@ -95,6 +95,7 @@ namespace ln.skyscanner.crawl.tests
}
catch (SocketException)
{
return false;
}
}
return false;

21
entities/GlobalNetwork.cs 100644
View File

@ -0,0 +1,21 @@
using System;
using System.Collections.Generic;
using System.Collections;
using System.Security;
namespace ln.skyscanner.entities
{
public class GlobalNetwork
{
public IEnumerable<Subnet> Subnets => subnets;
public IEnumerable<Node> Nodes => nodes;
private List<Node> nodes = new List<Node>();
private List<Subnet> subnets = new List<Subnet>();
public GlobalNetwork()
{
}
}
}

54
entities/NetworkInterface.cs 100644
View File

@ -0,0 +1,54 @@
// /**
// * File: Router.cs
// * Author: haraldwolff
// *
// * This file and it's content is copyrighted by the Author and / or copyright holder.
// * Any use wihtout proper permission is illegal and may lead to legal actions.
// *
// *
// **/
using System;
using System.Collections.Generic;
using ln.types;
using ln.skyscanner.crawl;
namespace ln.skyscanner.entities
{
public class NetworkInterface
{
public Node Node { get; private set; } = null;
public string Name { get; private set; } = "";
public CIDR[] IPs { get; set; } = new CIDR[0];
private NetworkInterface()
{ }
public NetworkInterface(Node node)
{
Node = node;
}
public NetworkInterface(Node node,String name)
{
Node = node;
Name = name;
}
public override int GetHashCode()
{
return Node.GetHashCode() ^ Name.GetHashCode();
}
public override bool Equals(object obj)
{
if (obj is NetworkInterface)
{
NetworkInterface networkInterface = obj as NetworkInterface;
return Node.Equals(networkInterface.Node) && Name.Equals(networkInterface.Name);
}
return false;
}
}
}

44
entities/Node.cs
View File

@ -10,27 +10,53 @@
using System;
using ln.types;
using System.Collections.Generic;
using System.Linq;
namespace ln.skyscanner.entities
{
public class Node
{
public CIDR PrimaryIP;
public String Name;
public GlobalNetwork GlobalNetwork { get; private set; }
public List<NetworkInterface> NetworkInterfaces = new List<NetworkInterface>();
public CIDR PrimaryIP { get; set; }
public String PrimaryMac { get; set; }
public String Name { get; set; }
public DateTime FirstSeen;
public DateTime LastSeen;
public DateTime Created { get; }
public Uri[] URIs => uris.ToArray();
public Node()
private HashSet<Uri> uris = new HashSet<Uri>();
private Node()
{
FirstSeen = DateTime.Now;
}
public Node(CIDR primaryIP)
public Node(GlobalNetwork globalNetwork, CIDR primaryIP)
{
GlobalNetwork = globalNetwork;
PrimaryIP = primaryIP;
FirstSeen = DateTime.Now;
Name = PrimaryIP.ToString();
Created = DateTime.Now;
}
public void RemoveURI(Uri uri)
{
foreach (Uri u in uris)
{
if (uri.Equals(u))
uris.Remove(u);
}
}
public void AddURI(Uri uri)
{
uris.Add(uri);
}
public IEnumerable<Uri> FindURIs(string scheme)
{
return uris.Where((u) => scheme.Equals(u.Scheme));
}
}

36
entities/Router.cs
View File

@ -1,36 +0,0 @@
// /**
// * File: Router.cs
// * Author: haraldwolff
// *
// * This file and it's content is copyrighted by the Author and / or copyright holder.
// * Any use wihtout proper permission is illegal and may lead to legal actions.
// *
// *
// **/
using System;
using System.Collections.Generic;
using ln.types;
namespace ln.skyscanner.entities
{
public class NetworkInterface
{
public Node Node;
public string Name;
public List<CIDR> IPs = new List<CIDR>();
public NetworkInterface()
{ }
public NetworkInterface(Node node)
{
Node = node;
}
public NetworkInterface(Node node,String name)
{
Node = node;
Name = name;
}
}
}

27
entities/Subnet.cs
View File

@ -9,10 +9,15 @@
// **/
using System;
using ln.types;
using ln.types.odb;
using System.Collections.Generic;
using System.Linq;
namespace ln.skyscanner.entities
{
public class Subnet
public class Subnet : Persistent
{
public GlobalNetwork GlobalNetwork { get; private set; }
public readonly CIDR Network;
public String Name;
@ -21,18 +26,32 @@ namespace ln.skyscanner.entities
public DateTime LastScan;
public DateTime NextScan;
public Subnet()
public NetworkInterface[] AttachedInterfaces => networkInterfaces.ToArray();
private HashSet<NetworkInterface> networkInterfaces = new HashSet<NetworkInterface>();
private Subnet()
{
FirstSeen = DateTime.Now;
}
public Subnet(CIDR cidr)
public Subnet(GlobalNetwork globalNetwork,CIDR cidr)
{
GlobalNetwork = globalNetwork;
Network = cidr;
Name = Network.ToString();
FirstSeen = DateTime.Now;
}
public void AttachInterface(NetworkInterface networkInterface)
{
networkInterfaces.Add(networkInterface);
}
public void DetachInterface(NetworkInterface networkInterface)
{
networkInterfaces.Remove(networkInterface);
}
public override string ToString()
{
return Network.ToString();

2
http/CrawlerApi.cs
View File

@ -20,7 +20,7 @@ namespace ln.skyscanner.http
}
[Callable]
public Subnet[] GetSubnets()
public CrawledSubnet[] GetSubnets()
{
return SkyScanner.Crawler?.CrawlPool.Subnets.ToArray();
}

4
ln.skyscanner.csproj
View File

@ -50,7 +50,7 @@
<Compile Include="crawl\Crawler.cs" />
<Compile Include="crawl\CrawledHost.cs" />
<Compile Include="entities\Subnet.cs" />
<Compile Include="entities\Router.cs" />
<Compile Include="entities\NetworkInterface.cs" />
<Compile Include="Check.cs" />
<Compile Include="check\Hostalive.cs" />
<Compile Include="SkyScanner.cs" />
@ -66,6 +66,8 @@
<Compile Include="crawl\tests\SNMP.cs" />
<Compile Include="crawl\tests\ICMP.cs" />
<Compile Include="crawl\tests\RFC1213.cs" />
<Compile Include="entities\GlobalNetwork.cs" />
<Compile Include="crawl\CrawledSubnet.cs" />
</ItemGroup>
<ItemGroup>
<None Include="packages.config" />

4
templates/static/crawlerhosts.html
View File

@ -117,8 +117,8 @@
{ title: "Zuletzt gesehen", data: "LastSeen", sorter: "", width: 180, formatter: "datetime", formatterParams: { inputFormat: "", outputFormat: "DD.MM.YYYY hh:mm:ss", invalidPlaceHolder: "-" } },
{ title: "Letzter Scan", data: "LastCheck", sorter: "", width: 180, formatter: "datetime", formatterParams: { inputFormat: "", outputFormat: "DD.MM.YYYY hh:mm:ss", invalidPlaceHolder: "-" } },
{ title: "Nächster Scan",data: "NextCheck",sorter: "",width: 180,formatter: "datetime",formatterParams: {inputFormat: "",outputFormat: "DD.MM.YYYY hh:mm:ss",invalidPlaceHolder: "-"} },
{ title: "SNMP", data: "SSHDetected", width: 90 },
{ title: "SSH", data: "SnmpDetected", width: 90 },
{ title: "SSH", data: "SSHDetected", width: 90 },
{ title: "SNMP", data: "SnmpDetected", width: 90 },
{ title: "RFC1213", data: "RFC1213Detected", width: 90 }
];