Skytron
/
ln.skyscanner
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

WIP

broken
Harald Wolff 2019-04-02 01:25:44 +02:00
parent 4cac9fbf56
commit 5e514b5afc
22 changed files with 2835 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
Program.cs
View File

@ -8,33 +8,10 @@
// * // *
// **/ // **/
using System; using System;
using Renci.SshNet;
using ln.skyscanner.identify;
using System.Net;
using System.Collections.Generic; using System.Collections.Generic;
using ln.snmp.types;
using System.IO;
using ln.snmp; using ln.snmp;
using ln.snmp.endpoint;
using System.Diagnostics;
using ln.perfdb.storage;
using ln.logging; using ln.logging;
using System.Security.Cryptography;
using ln.snmp.asn1;
using System.Linq;
using ln.types;
using System.Runtime.InteropServices;
using ln.snmp.rfc1213;
using ln.types.sync;
using ln.skyscanner.crawl;
using System.Threading;
using ln.skyscanner.crawl.tests;
using Castle.DynamicProxy;
using ln.types.odb;
using System.Net.Sockets;
using ln.skyscanner.entities; using ln.skyscanner.entities;
using ln.types.btree;
using ln.types.test;
using ln.types.threads; using ln.types.threads;
using ln.types.net; using ln.types.net;
@ -52,7 +29,7 @@ namespace ln.skyscanner
fileLogger.MaxLogLevel = LogLevel.INFO; fileLogger.MaxLogLevel = LogLevel.INFO;
Logger.Default.Backends.Add(fileLogger); Logger.Default.Backends.Add(fileLogger);
Logger.ConsoleLogger.MaxLogLevel = LogLevel.DEBUGDETAIL; //Logger.ConsoleLogger.MaxLogLevel = LogLevel.DEBUGDETAIL;
Initialize(); Initialize();
@ -60,32 +37,31 @@ namespace ln.skyscanner
skyScanner.Start(); skyScanner.Start();
return; return;
foreach (IntfIP iip in SkyScanner.Instance.Entities.intfIPCollection.Select("Network", Network4.Parse("10.255.9.16/29")))
{
Console.WriteLine("INTF-IP ():{0}", iip);
}
foreach (IntfIP iip in SkyScanner.Instance.Entities.intfIPCollection)
{
Console.WriteLine("INTF-IP: {0}", iip);
}
IEnumerable<Node> nodes = skyScanner.Entities.nodeCollection; IEnumerable<Node> nodes = skyScanner.Entities.nodeCollection;
foreach (Node node in nodes) foreach (Node node in nodes)
{ {
Console.WriteLine("Node: {0}", node); Console.WriteLine("Node: {0}", node);
skyScanner.Entities.nodeCollection.Update(node);
foreach (Node checkNode in SkyScanner.Instance.Entities.nodeCollection.Select("PrimaryIP", node.PrimaryIP)) Timing.Meassure("", () =>
{ {
Console.WriteLine("Node (lookup primary ip): {0}", checkNode); foreach (Node neighbor in SkyScanner.Instance.Entities.GlobalNetwork.FindNeighbors(node))
if (!checkNode.ID.Equals(node)) {
{ Console.WriteLine(" Neighbor: {0}", neighbor);
if ((checkNode.Interfaces.Count() == 0)) }
{ });
SkyScanner.Instance.Entities.nodeCollection.Delete(checkNode);
}
}
}
Console.WriteLine("");
//IEnumerable<NetworkInterface> interfaces = Timing.Meassure("node.Interfaces",() => node.Interfaces);
//foreach (NetworkInterface intf in interfaces)
//{
// Console.WriteLine(" Interface: {0}", intf);
//}
Console.WriteLine("");
Console.WriteLine(""); Console.WriteLine("");
} }

1
SkyEntities.cs
View File

@ -40,6 +40,7 @@ namespace ln.skyscanner
nodeCollection.EnsureIndex("PrimaryIP"); nodeCollection.EnsureIndex("PrimaryIP");
interfaceCollection.EnsureIndex("NodeID"); interfaceCollection.EnsureIndex("NodeID");
intfIPCollection.EnsureIndex("interfaceID"); intfIPCollection.EnsureIndex("interfaceID");
intfIPCollection.EnsureIndex("Network");
GlobalNetwork = new GlobalNetwork(this); GlobalNetwork = new GlobalNetwork(this);
} }

4
crawl/Crawl.cs
View File

@ -100,10 +100,8 @@ namespace ln.skyscanner.crawl
try try
{ {
while (servicesToCheck.Count > 0) while (servicesToCheck.Count > 0)
RunServiceCheck(servicesToCheck[0]); RunServiceCheck(servicesToCheck[0]);
} catch (Exception e) } catch (Exception e)
{ {
Logging.Log(LogLevel.ERROR, "Crawl.RunJob(): Caught exception: {0}",e); Logging.Log(LogLevel.ERROR, "Crawl.RunJob(): Caught exception: {0}",e);
@ -112,7 +110,7 @@ namespace ln.skyscanner.crawl
Host.LastCheckTime = DateTime.Now - dateTime; Host.LastCheckTime = DateTime.Now - dateTime;
Host.LastCheck = dateTime; Host.LastCheck = dateTime;
Host.NextCheck = dateTime + TimeSpan.FromHours(1); Host.NextCheck = dateTime + TimeSpan.FromHours(24);
bool updated = Crawler.CrawledHosts.Upsert( Host ); bool updated = Crawler.CrawledHosts.Upsert( Host );

11
crawl/CrawlNetwork.cs
View File

@ -16,6 +16,7 @@ using ln.logging;
using Newtonsoft.Json; using Newtonsoft.Json;
using System.Linq; using System.Linq;
using ln.types.odb; using ln.types.odb;
using ln.types.net;
namespace ln.skyscanner.crawl namespace ln.skyscanner.crawl
{ {
public class CrawlNetwork : PoolJob, IPersistent public class CrawlNetwork : PoolJob, IPersistent
@ -24,7 +25,7 @@ namespace ln.skyscanner.crawl
public Crawler Crawler { get; } public Crawler Crawler { get; }
public CrawledSubnet Subnet { get; } public CrawledSubnet Subnet { get; }
public CIDR Network { get; } public Network4 Network { get; }
@ -36,7 +37,7 @@ namespace ln.skyscanner.crawl
Name = String.Format("Subnet crawl {0}", subnet.Network); Name = String.Format("Subnet crawl {0}", subnet.Network);
} }
public CrawlNetwork(Crawler crawler, CrawledSubnet subnet,CIDR network) public CrawlNetwork(Crawler crawler, CrawledSubnet subnet,Network4 network)
{ {
Crawler = crawler; Crawler = crawler;
Subnet = subnet; Subnet = subnet;
@ -47,9 +48,9 @@ namespace ln.skyscanner.crawl
public override void RunJob() public override void RunJob()
{ {
if (Network.MaskWidth < 26) if (Network.Width < 26)
{ {
SplitJob(Network.Divide(26 - Network.MaskWidth).Select((n) => new CrawlNetwork(Crawler, Subnet, n)).ToArray()); SplitJob(Network.Divide(26 - (int)Network.Width).Select((n) => new CrawlNetwork(Crawler, Subnet, n)).ToArray());
return; return;
} }
@ -62,7 +63,7 @@ namespace ln.skyscanner.crawl
int count = 0; int count = 0;
foreach (CIDR ip in Network) foreach (IPv4 ip in Network)
{ {
count++; count++;

16
crawl/CrawledHost.cs
View File

@ -15,6 +15,7 @@ using System.Linq;
using ln.snmp.rfc1213; using ln.snmp.rfc1213;
using LiteDB; using LiteDB;
using ln.types.odb; using ln.types.odb;
using ln.types.net;
namespace ln.skyscanner.crawl namespace ln.skyscanner.crawl
{ {
@ -24,13 +25,14 @@ namespace ln.skyscanner.crawl
public readonly Guid ID = Guid.NewGuid(); public readonly Guid ID = Guid.NewGuid();
public String Name { get; set; } public String Name { get; set; }
public CIDR PrimaryIP { get; set; } public IPv4 PrimaryIP { get; set; }
public String[] HWAddresses = new string[0]; public String[] HWAddresses = new string[0];
public String PrimaryHWAddr => HWAddresses.Where((hwa) => hwa != null && !String.Empty.Equals(hwa)).FirstOrDefault(); public String PrimaryHWAddr => HWAddresses.Where((hwa) => hwa != null && !String.Empty.Equals(hwa)).FirstOrDefault();
public CIDR[] IPAddresses { get; set; } = new CIDR[0]; public IPv4[] IPAddresses { get; set; } = new IPv4[0];
public Network4[] Networks { get; set; } = new Network4[0];
Dictionary<string, object> hints = new Dictionary<string, object>(); Dictionary<string, object> hints = new Dictionary<string, object>();
public DateTime FirstSeen { get; set; } public DateTime FirstSeen { get; set; }
@ -46,13 +48,13 @@ namespace ln.skyscanner.crawl
public CrawledHost() public CrawledHost()
{ {
Name = "noname"; Name = "noname";
PrimaryIP = IPAddress.Any; PrimaryIP = IPv4.ANY;
} }
public bool HasIP(CIDR ip) public bool HasIP(IPv4 ip)
{ {
foreach (CIDR _ip in IPAddresses) foreach (IPv4 _ip in IPAddresses)
if (_ip.Host.Equals(ip)) if (_ip.Equals(ip))
return true; return true;
return false; return false;
} }

7
crawl/CrawledSubnet.cs
View File

@ -12,6 +12,7 @@ using ln.types;
using ln.types.odb; using ln.types.odb;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using ln.types.net;
namespace ln.skyscanner.crawl namespace ln.skyscanner.crawl
{ {
public class CrawledSubnet public class CrawledSubnet
@ -19,7 +20,7 @@ namespace ln.skyscanner.crawl
[DocumentID] [DocumentID]
Guid ID = Guid.NewGuid(); Guid ID = Guid.NewGuid();
public readonly CIDR Network; public readonly Network4 Network;
public String Name; public String Name;
public DateTime FirstSeen; public DateTime FirstSeen;
@ -33,9 +34,9 @@ namespace ln.skyscanner.crawl
FirstSeen = DateTime.Now; FirstSeen = DateTime.Now;
} }
public CrawledSubnet(CIDR cidr) public CrawledSubnet(Network4 network)
{ {
Network = cidr; Network = network;
Name = Network.ToString(); Name = Network.ToString();
FirstSeen = DateTime.Now; FirstSeen = DateTime.Now;
} }

15
crawl/Crawler.cs
View File

@ -31,6 +31,7 @@ using ln.types.odb;
using ln.skyscanner.crawl.service; using ln.skyscanner.crawl.service;
using ln.skyscanner.crawl.tests; using ln.skyscanner.crawl.tests;
using LiteDB; using LiteDB;
using ln.types.net;
namespace ln.skyscanner.crawl namespace ln.skyscanner.crawl
{ {
@ -41,6 +42,7 @@ namespace ln.skyscanner.crawl
CrawlService.RegisterService(new TCP(new int[] { 13080, 13022, 80, 22, 443, 13443 })); CrawlService.RegisterService(new TCP(new int[] { 13080, 13022, 80, 22, 443, 13443 }));
CrawlService.RegisterService(new SNMP(new string[] { "VhclfC7lfIojYZ", "Vhclf(C7$lfIojYZ", "ByFR4oW98hap", "qVy3hnZJ2fov" })); CrawlService.RegisterService(new SNMP(new string[] { "VhclfC7lfIojYZ", "Vhclf(C7$lfIojYZ", "ByFR4oW98hap", "qVy3hnZJ2fov" }));
CrawlService.RegisterService(new RFC1213()); CrawlService.RegisterService(new RFC1213());
CrawlService.RegisterService(new HTTP());
} }
public SkyScanner SkyScanner { get; } public SkyScanner SkyScanner { get; }
@ -67,12 +69,12 @@ namespace ln.skyscanner.crawl
public ODBCollection<CrawledHost> CrawledHosts => hosts; public ODBCollection<CrawledHost> CrawledHosts => hosts;
public ODBCollection<CrawledSubnet> CrawledSubnets => subnets; public ODBCollection<CrawledSubnet> CrawledSubnets => subnets;
public ODBCollection<Network4> BlockedNetworks => blockedNetworks;
//private ODB<CrawlPool> dbCrawlPool;
private ODB odbDatabase; private ODB odbDatabase;
private ODBCollection<CrawledHost> hosts; private ODBCollection<CrawledHost> hosts;
private ODBCollection<CrawledSubnet> subnets; private ODBCollection<CrawledSubnet> subnets;
private ODBCollection<Network4> blockedNetworks;
public Crawler(SkyScanner skyScanner) public Crawler(SkyScanner skyScanner)
@ -94,6 +96,7 @@ namespace ln.skyscanner.crawl
odbDatabase = new ODB(BasePath); odbDatabase = new ODB(BasePath);
hosts = odbDatabase.GetCollection<CrawledHost>(); hosts = odbDatabase.GetCollection<CrawledHost>();
subnets = odbDatabase.GetCollection<CrawledSubnet>(); subnets = odbDatabase.GetCollection<CrawledSubnet>();
blockedNetworks = odbDatabase.GetCollection<Network4>("blockedNetworks");
CrawlerState = ComponentState.INITIALIZED; CrawlerState = ComponentState.INITIALIZED;
} }
@ -168,7 +171,7 @@ namespace ln.skyscanner.crawl
} }
} }
public void EnsureSubnet(CIDR network) public void EnsureSubnet(Network4 network)
{ {
FindSubnet(network); FindSubnet(network);
} }
@ -195,7 +198,7 @@ namespace ln.skyscanner.crawl
crawlThreadPool.Enqueue(crawlSubnet); crawlThreadPool.Enqueue(crawlSubnet);
} }
public CrawledHost FindHostForIP(CIDR ip) public CrawledHost FindHostForIP(IPv4 ip)
{ {
CrawledHost crawledHost = CrawledHosts.Where(host => host.HasIP(ip)).FirstOrDefault(); CrawledHost crawledHost = CrawledHosts.Where(host => host.HasIP(ip)).FirstOrDefault();
if (crawledHost == null) if (crawledHost == null)
@ -208,7 +211,7 @@ namespace ln.skyscanner.crawl
} }
return crawledHost; return crawledHost;
} }
public CrawledSubnet FindSubnet(CIDR network) public CrawledSubnet FindSubnet(Network4 network)
{ {
CrawledSubnet sn = CrawledSubnets.Where(subnet => subnet.Network.Equals(network)).FirstOrDefault(); CrawledSubnet sn = CrawledSubnets.Where(subnet => subnet.Network.Equals(network)).FirstOrDefault();
if (sn == null) if (sn == null)
@ -238,7 +241,7 @@ namespace ln.skyscanner.crawl
} }
if (CrawlSubnets) if (CrawlSubnets)
foreach (CrawledSubnet subnet in CrawledSubnets.Where( sn => (sn.NextScan < (DateTime.Now - TimeSpan.FromDays(1)) && sn.Network.MaskWidth >= 24))) foreach (CrawledSubnet subnet in CrawledSubnets.Where( sn => (sn.NextScan < DateTime.Now ) && sn.Network.Width >= 24))
{ {
Crawl(subnet); Crawl(subnet);
} }

92
crawl/service/HTTP.cs 100644
View File

@ -0,0 +1,92 @@
// /**
// * File: HTTP.cs
// * Author: haraldwolff
// *
// * This file and it's content is copyrighted by the Author and / or copyright holder.
// * Any use wihtout proper permission is illegal and may lead to legal actions.
// *
// *
// **/
using System;
using System.Net.Http;
using ln.logging;
using ln.types.net;
using System.Linq;
using System.Net;
using System.IO;
namespace ln.skyscanner.crawl.service
{
public class HTTP : CrawlService
{
public HTTP()
:base("http")
{
}
public override bool Check(Crawl crawl)
{
foreach (int port in new int[] { 13080, 80 })
{
if (crawl.CheckRequiredOption("tcp", port))
{
if (CheckPort(crawl, port))
return true;
}
}
return false;
}
private bool CheckPort(Crawl crawl,int port)
{
foreach (IPv4 ip in crawl.Host.IPAddresses.Concat(new IPv4[] { crawl.Host.PrimaryIP }).Distinct())
{
try
{
crawl.setState("Check HTTP at {0}:{1}",ip,port);
HttpWebRequest webRequest = WebRequest.CreateHttp(String.Format("http://{0}:{1}", ip, port));
webRequest.AllowAutoRedirect = true;
HttpWebResponse response = (HttpWebResponse)webRequest.GetResponse();
String content = null;
using (StreamReader reader = new StreamReader(response.GetResponseStream()))
{
content = reader.ReadToEnd();
reader.Close();
}
crawl.Host.SetHint("http.ip", ip);
crawl.Host.SetHint("http.port", port);
crawl.Host.SetHint("http.status", response.StatusCode);
crawl.Host.SetHint("http.server", response.Server);
crawl.Host.SetHint("http.content", content);
return true;
}
catch (WebException e)
{
Logging.Log(e);
}
}
return false;
}
public override bool HostProvidesOption(Crawl crawl, params object[] parameters)
{
if (parameters.Length == 0)
return false;
foreach (object oport in parameters)
{
int port = (int)oport;
if (!crawl.Host.GetHint(String.Format("http.{0}", port), false))
return false;
}
return true;
}
}
}

7
crawl/service/RFC1213.cs
View File

@ -49,7 +49,12 @@ namespace ln.skyscanner.crawl.tests
crawl.Host.IPAddresses = interfaces.SelectMany(intf => intf.IPAddresses).ToArray(); crawl.Host.IPAddresses = interfaces.SelectMany(intf => intf.IPAddresses).ToArray();
crawl.Host.HWAddresses = interfaces.Select(intf => intf.HWAddr).ToArray(); crawl.Host.HWAddresses = interfaces.Select(intf => intf.HWAddr).ToArray();
crawl.Host.SetHint("rfc1213.interfaces",interfaces.Select(intf => string.Format("{0};{1};{2}",intf.Name,intf.HWAddr,String.Join(",",intf.IPAddresses.Select(ip=>ip.ToString())))).ToArray()); crawl.Host.SetHint("rfc1213.interfaces", interfaces.Select(intf => string.Format("{0};{1};{2}",
intf.Name,
intf.HWAddr,
String.Join(",", intf.IPAddresses.Zip(intf.Netmasks,(ip,mask) => String.Format("{0}/{1}",ip.ToString(),mask.ToString())))
)).ToArray()
);
crawl.Host.SetHint("rfc1213", true); crawl.Host.SetHint("rfc1213", true);
} }
} catch (TimeoutException) } catch (TimeoutException)

17
crawl/service/SNMP.cs
View File

@ -16,6 +16,7 @@ using ln.snmp.types;
using System.Runtime.Remoting.Messaging; using System.Runtime.Remoting.Messaging;
using ln.skyscanner.crawl.service; using ln.skyscanner.crawl.service;
using System.Linq; using System.Linq;
using ln.types.net;
namespace ln.skyscanner.crawl.tests namespace ln.skyscanner.crawl.tests
{ {
public class SNMP : CrawlService public class SNMP : CrawlService
@ -71,9 +72,9 @@ namespace ln.skyscanner.crawl.tests
private bool TestSnmpV3(CrawledHost crawledHost) private bool TestSnmpV3(CrawledHost crawledHost)
{ {
CIDR[] ips = crawledHost.IPAddresses.Concat(new CIDR[] { crawledHost.PrimaryIP }).Distinct().ToArray(); IPv4[] ips = crawledHost.IPAddresses.Concat(new IPv4[] { crawledHost.PrimaryIP }).Distinct().ToArray();
foreach (CIDR ip in ips) foreach (IPv4 ip in ips)
{ {
using (USMEndpoint v3endpoint = new USMEndpoint(SNMPEngine.DefaultEngine, new IPEndPoint(ip, 161))) using (USMEndpoint v3endpoint = new USMEndpoint(SNMPEngine.DefaultEngine, new IPEndPoint(ip, 161)))
{ {
@ -89,7 +90,6 @@ namespace ln.skyscanner.crawl.tests
{ {
crawledHost.SetHint("snmp.version", 3); crawledHost.SetHint("snmp.version", 3);
bool replied = false;
int c = 0; int c = 0;
foreach (string community in defaultCommunities) foreach (string community in defaultCommunities)
@ -114,6 +114,7 @@ namespace ln.skyscanner.crawl.tests
} }
catch (TimeoutException) catch (TimeoutException)
{ {
return false;
} }
catch (SnmpError) catch (SnmpError)
{ {
@ -128,9 +129,9 @@ namespace ln.skyscanner.crawl.tests
private bool TestSnmpV2(CrawledHost crawledHost) private bool TestSnmpV2(CrawledHost crawledHost)
{ {
CIDR[] ips = crawledHost.IPAddresses.Concat(new CIDR[] { crawledHost.PrimaryIP }).Distinct().ToArray(); IPv4[] ips = crawledHost.IPAddresses.Concat(new IPv4[] { crawledHost.PrimaryIP }).Distinct().ToArray();
foreach (CIDR ip in ips) foreach (IPv4 ip in ips)
{ {
using (SnmpV2Endpoint v2endpoint = new SnmpV2Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(crawledHost.PrimaryIP, 161))) using (SnmpV2Endpoint v2endpoint = new SnmpV2Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(crawledHost.PrimaryIP, 161)))
{ {
@ -162,9 +163,9 @@ namespace ln.skyscanner.crawl.tests
private bool TestSnmpV1(CrawledHost crawledHost) private bool TestSnmpV1(CrawledHost crawledHost)
{ {
CIDR[] ips = crawledHost.IPAddresses.Concat(new CIDR[] { crawledHost.PrimaryIP }).Distinct().ToArray(); IPv4[] ips = crawledHost.IPAddresses.Concat(new IPv4[] { crawledHost.PrimaryIP }).Distinct().ToArray();
foreach (CIDR ip in ips) foreach (IPv4 ip in ips)
{ {
using (SnmpV1Endpoint v1endpoint = new SnmpV1Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(crawledHost.PrimaryIP, 161))) using (SnmpV1Endpoint v1endpoint = new SnmpV1Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(crawledHost.PrimaryIP, 161)))
{ {
@ -237,7 +238,7 @@ namespace ln.skyscanner.crawl.tests
public static SnmpInterface GetSnmpInterface(CrawledHost crawledHost) public static SnmpInterface GetSnmpInterface(CrawledHost crawledHost)
{ {
int snmpVersion = crawledHost.GetHint<int>("snmp.version", -1); int snmpVersion = crawledHost.GetHint<int>("snmp.version", -1);
CIDR snmpAddress = crawledHost.GetHint<CIDR>("snmp.address", crawledHost.PrimaryIP); IPv4 snmpAddress = crawledHost.GetHint<IPv4>("snmp.address", crawledHost.PrimaryIP);
switch (snmpVersion) switch (snmpVersion)
{ {

11
crawl/service/SSH.cs
View File

@ -12,6 +12,7 @@ using Renci.SshNet;
using ln.types; using ln.types;
using Renci.SshNet.Common; using Renci.SshNet.Common;
using System.Net.Sockets; using System.Net.Sockets;
using ln.types.net;
namespace ln.skyscanner.crawl.tests namespace ln.skyscanner.crawl.tests
{ {
public static class SSH public static class SSH
@ -37,11 +38,11 @@ namespace ln.skyscanner.crawl.tests
public static bool CanConnect(CrawledHost crawledHost) public static bool CanConnect(CrawledHost crawledHost)
{ {
int sshPort = crawledHost.GetHint<int>("ssh.port", -1); int sshPort = crawledHost.GetHint<int>("ssh.port", -1);
CIDR sshIP = crawledHost.GetHint<CIDR>("ssh.ip", null); IPv4 sshIP = crawledHost.GetHint<IPv4>("ssh.ip", null);
string sshUser = crawledHost.GetHint<string>("ssh.login", null); string sshUser = crawledHost.GetHint<string>("ssh.login", null);
string sshPassword = crawledHost.GetHint<string>("ssh.password", null); string sshPassword = crawledHost.GetHint<string>("ssh.password", null);
if ((sshPort == -1) || !CanConnect(crawledHost,sshIP.Host,sshPort,sshUser,sshPassword)) if ((sshPort == -1) || !CanConnect(crawledHost,sshIP,sshPort,sshUser,sshPassword))
{ {
if (!Scan(crawledHost)) if (!Scan(crawledHost))
{ {
@ -58,7 +59,7 @@ namespace ln.skyscanner.crawl.tests
private static bool Scan(CrawledHost crawledHost) private static bool Scan(CrawledHost crawledHost)
{ {
foreach (CIDR ip in crawledHost.IPAddresses) foreach (IPv4 ip in crawledHost.IPAddresses)
{ {
foreach (int port in new int[] { 13022, 22 }) foreach (int port in new int[] { 13022, 22 })
{ {
@ -66,7 +67,7 @@ namespace ln.skyscanner.crawl.tests
{ {
foreach (string password in defaultPasswords) foreach (string password in defaultPasswords)
{ {
if (CanConnect(crawledHost, ip.Host, port, "skytron", password, true)) if (CanConnect(crawledHost, ip, port, "skytron", password, true))
return true; return true;
} }
} catch (SocketException) } catch (SocketException)
@ -78,7 +79,7 @@ namespace ln.skyscanner.crawl.tests
return false; return false;
} }
private static bool CanConnect(CrawledHost crawledHost, CIDR host, int port, string username, string password,bool throwe = false) private static bool CanConnect(CrawledHost crawledHost, IPv4 host, int port, string username, string password,bool throwe = false)
{ {
using (TcpClient tcp = new TcpClient()) using (TcpClient tcp = new TcpClient())
{ {

104
entities/GlobalNetwork.cs
View File

@ -40,7 +40,7 @@ namespace ln.skyscanner.entities
SkyEntities = skyEntities; SkyEntities = skyEntities;
} }
public void EnsureSubnet(CIDR network) public void EnsureSubnet(Network4 network)
{ {
SkyScanner.Crawler.EnsureSubnet(network); SkyScanner.Crawler.EnsureSubnet(network);
@ -58,35 +58,39 @@ namespace ln.skyscanner.entities
return SkyEntities.nodeCollection.Select(id); return SkyEntities.nodeCollection.Select(id);
} }
public Subnet FindSubnetForHost(CIDR host) public Subnet FindSubnetForIP(IPv4 ip)
{ {
return SkyEntities.subnetCollection.Select(net => net.Network.Contains(host)).FirstOrDefault(); return SkyEntities.subnetCollection.Select(net => net.Network.Contains(ip)).FirstOrDefault();
} }
public IEnumerable<Node> FindHostsInSubnet(CIDR network) public IEnumerable<Node> FindHostsInSubnet(Network4 network)
{ {
IntfIP[] intfIPs = SkyEntities.intfIPCollection.Select((ip) => ip.IP.In(network)).ToArray(); IntfIP[] intfIPs = SkyEntities.intfIPCollection.Select("Network",network).ToArray();
NetworkInterface[] networkInterfaces = intfIPs.Select((iip) => iip.NetworkInterface).Where((i)=>i!=null).Distinct().ToArray(); NetworkInterface[] networkInterfaces = intfIPs.Select((iip) => iip.NetworkInterface).ToArray();
Node[] nodes = SkyEntities.nodeCollection.Select(networkInterfaces.Select((ni) => new ODBGuid(ni.NodeID))).ToArray(); Node[] nodes = SkyEntities.nodeCollection.Select(networkInterfaces.Select((ni) => new ODBGuid(ni.NodeID))).ToArray();
return nodes; return nodes;
} }
public IEnumerable<Node> FindNeighbors(Node node) public IEnumerable<Node> FindNeighbors(Node node)
{ {
HashSet<Node> nodes = new HashSet<Node>(); HashSet<Node> nodes = new HashSet<Node>();
foreach (Node neighbor in node.IPAdresses.SelectMany(
(ip) => Timing.Meassure("FindHostsInSubnet", () => foreach (Network4 network in node.Networks)
FindHostsInSubnet(ip) {
) foreach (Node neighbor in FindHostsInSubnet(network))
)) {
nodes.Add(neighbor); nodes.Add(neighbor);
}
}
return nodes; return nodes;
} }
public Node FindNodeByIP(IEnumerable<CIDR> ips) public Node FindNodeByIP(IEnumerable<IPv4> ips)
{ {
foreach (CIDR ip in ips) foreach (IPv4 ip in ips)
{ {
Node node = FindNodeByIP(ip); Node node = FindNodeByIP(ip);
if (node != null) if (node != null)
@ -94,17 +98,17 @@ namespace ln.skyscanner.entities
} }
return null; return null;
} }
public Node FindNodeByIP(CIDR ip) //public Node FindNodeByIP(CIDR ip)
{ //{
ip = ip.Host; // ip = ip.Host;
/* // /*
* ToDo: Alternative Implementation // * ToDo: Alternative Implementation
* a) higher performance (use indexing) // * a) higher performance (use indexing)
* b) fix bug: Node not found if no interface ips exist... // * b) fix bug: Node not found if no interface ips exist...
*/ // */
IntfIP _iip = SkyEntities.intfIPCollection.Where((iip) => iip.IP.Host.Equals(ip)).FirstOrDefault(); // IntfIP _iip = SkyEntities.intfIPCollection.Where((iip) => iip.IP.Host.Equals(ip)).FirstOrDefault();
return _iip?.NetworkInterface?.Node; // return _iip?.NetworkInterface?.Node;
} //}
public Node FindNodeByIP(IPv4 ip) public Node FindNodeByIP(IPv4 ip)
{ {
@ -196,30 +200,29 @@ namespace ln.skyscanner.entities
SkyEntities.interfaceCollection.Upsert(networkInterface); SkyEntities.interfaceCollection.Upsert(networkInterface);
CIDR[] crawledIPs = fields[2].Split(',').Where((sip) => !String.Empty.Equals(sip)).Select((sip) => CIDR.Parse(sip)).ToArray(); IPv4[][] crawledIPs = fields[2].Split(',').Where((sip) => !String.Empty.Equals(sip))
IntfIP[] currentIPs = networkInterface.IPs.ToArray(); .Select((sip) => sip.Split('/').Select(ip => IPv4.Parse(ip)).ToArray()).ToArray();
foreach (CIDR crawledIP in crawledIPs) IntfIP[] currentIPs = networkInterface.ConfiguredIPs.ToArray();
foreach (IPv4[] crawledIP in crawledIPs)
{ {
int n; int n;
for (n=0; n < currentIPs.Length; n++) for (n=0; n < currentIPs.Length; n++)
if (currentIPs[n].IP.Equals(crawledIP)) if (currentIPs[n].IP.Equals(crawledIP[0]))
break; break;
if (n == currentIPs.Length) if (n == currentIPs.Length)
{ {
IntfIP miss = new IntfIP(networkInterface, crawledIP); IntfIP miss = new IntfIP(networkInterface, crawledIP[0], new Network4(crawledIP[0], crawledIP[1]));
SkyEntities.intfIPCollection.Upsert(miss); SkyEntities.intfIPCollection.Upsert(miss);
} }
} }
foreach (IntfIP iip in currentIPs) foreach (IntfIP iip in currentIPs)
{ {
CIDR ip = iip.IP; IPv4 ip = iip.IP;
if (crawledIPs.Contains(ip)) if (!crawledIPs.Select((ipl)=>ipl[0]).Contains(ip))
{
}
else
{ {
SkyEntities.intfIPCollection.Delete(iip); SkyEntities.intfIPCollection.Delete(iip);
} }
@ -234,7 +237,7 @@ namespace ln.skyscanner.entities
String.Format("ssh://{0}:{1}@{2}:{3}", String.Format("ssh://{0}:{1}@{2}:{3}",
crawledHost.GetHint<string>("ssh.login"), crawledHost.GetHint<string>("ssh.login"),
crawledHost.GetHint<string>("ssh.password"), crawledHost.GetHint<string>("ssh.password"),
crawledHost.GetHint<CIDR>("ssh.ip"), crawledHost.GetHint<IPv4>("ssh.ip"),
crawledHost.GetHint<int>("ssh.port") crawledHost.GetHint<int>("ssh.port")
) )
) )
@ -242,11 +245,36 @@ namespace ln.skyscanner.entities
} }
foreach (CIDR ip in node.IPAdresses) node.RemoveURI("http");
if (crawledHost.GetHint("http.port",-1) != -1)
{ {
EnsureSubnet(ip.Network); node.AddURI(new URI(String.Format("http://{0}:{1}",
crawledHost.GetHint<IPv4>("http.ip", null),
crawledHost.GetHint<int>("http.port"))
));
} }
string httpIndex = crawledHost.GetHint("http.content", "");
if (httpIndex.Contains("<title>RouterOS router configuration page</title>"))
{
node.Vendor = "MicroTik";
}
if (crawledHost.GetHint("snmp.orids",new string[0]).Contains("1.3.6.1.4.1.41112"))
{
node.Vendor = "Ubiquity";
}
foreach (Network4 network in node.Networks)
EnsureSubnet(network);
node.LastUpdate = DateTime.Now; node.LastUpdate = DateTime.Now;
SkyEntities.nodeCollection.Upsert(node); SkyEntities.nodeCollection.Upsert(node);
} }

12
entities/IntfIP.cs
View File

@ -11,6 +11,7 @@ using System;
using ln.types.odb; using ln.types.odb;
using ln.types; using ln.types;
using Newtonsoft.Json; using Newtonsoft.Json;
using ln.types.net;
namespace ln.skyscanner.entities namespace ln.skyscanner.entities
{ {
public class IntfIP public class IntfIP
@ -23,18 +24,23 @@ namespace ln.skyscanner.entities
[JsonIgnore] [JsonIgnore]
public NetworkInterface NetworkInterface => SkyScanner.Instance.Entities.interfaceCollection.Select(interfaceID); public NetworkInterface NetworkInterface => SkyScanner.Instance.Entities.interfaceCollection.Select(interfaceID);
public CIDR IP { get; private set; } public IPv4 IP { get; private set; }
public Network4 Network { get; private set; }
public IntfIP() public IntfIP()
{ {
} }
public IntfIP(NetworkInterface networkInterface,CIDR ip) public IntfIP(NetworkInterface networkInterface,IPv4 ip,Network4 network)
{ {
interfaceID = networkInterface.ID; interfaceID = networkInterface.ID;
IP = ip; IP = ip;
Network = network;
} }
public override string ToString()
{
return string.Format("[IntfIP ID={0} IP={1,18} Network={2}]", ID, IP, Network);
}
} }

12
entities/NetworkInterface.cs
View File

@ -15,6 +15,7 @@ using ln.types.odb;
using Newtonsoft.Json; using Newtonsoft.Json;
using LiteDB; using LiteDB;
using ln.logging; using ln.logging;
using ln.types.net;
namespace ln.skyscanner.entities namespace ln.skyscanner.entities
{ {
public class NetworkInterface public class NetworkInterface
@ -34,8 +35,7 @@ namespace ln.skyscanner.entities
public string Name { get; private set; } = ""; public string Name { get; private set; } = "";
public string HWAddress { get; set; } = ""; public string HWAddress { get; set; } = "";
//public CIDR[] IPs { get; set; } = new CIDR[0]; public IEnumerable<IntfIP> ConfiguredIPs => SkyScanner.Instance.Entities.intfIPCollection.Select("interfaceID", ID);
public IEnumerable<IntfIP> IPs => SkyScanner.Instance.Entities.intfIPCollection.Select("interfaceID", ID);
private NetworkInterface() private NetworkInterface()
{ } { }
@ -46,13 +46,11 @@ namespace ln.skyscanner.entities
Name = name; Name = name;
} }
public bool HasIP(CIDR ip) public bool HasIP(IPv4 ip)
{ {
ip = ip.Host; foreach (IntfIP ifip in ConfiguredIPs)
foreach (IntfIP ifip in IPs)
{ {
if (ifip.IP.Host.Equals(ip)) if (ifip.IP.Equals(ip))
return true; return true;
} }
return false; return false;

8
entities/Node.cs
View File

@ -39,8 +39,12 @@ namespace ln.skyscanner.entities
public string ProductLine { get; set; } public string ProductLine { get; set; }
[JsonIgnore]
public IEnumerable<NetworkInterface> Interfaces => SkyScanner.Instance.Entities.interfaceCollection.Select("NodeID", ID); public IEnumerable<NetworkInterface> Interfaces => SkyScanner.Instance.Entities.interfaceCollection.Select("NodeID", ID);
public IEnumerable<CIDR> IPAdresses => Interfaces.SelectMany(intf => intf.IPs).Select((iip) => iip.IP).Distinct(); [JsonIgnore]
public IEnumerable<IPv4> IPAdresses => Interfaces.SelectMany(intf => intf.ConfiguredIPs.Select(nip => nip.IP));
[JsonIgnore]
public IEnumerable<Network4> Networks => Interfaces.SelectMany(intf => intf.ConfiguredIPs.Select(nip => nip.Network));
private HashSet<URI> uris = new HashSet<URI>(); private HashSet<URI> uris = new HashSet<URI>();
@ -48,7 +52,7 @@ namespace ln.skyscanner.entities
{ {
} }
public Node(CIDR primaryIP) public Node(IPv4 primaryIP)
{ {
PrimaryIP = primaryIP; PrimaryIP = primaryIP;
Name = PrimaryIP.ToString(); Name = PrimaryIP.ToString();

9
entities/Subnet.cs
View File

@ -12,6 +12,7 @@ using ln.types;
using ln.types.odb; using ln.types.odb;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using ln.types.net;
namespace ln.skyscanner.entities namespace ln.skyscanner.entities
{ {
public class Subnet : Persistent public class Subnet : Persistent
@ -21,7 +22,7 @@ namespace ln.skyscanner.entities
[DocumentID] [DocumentID]
public Guid ID { get; private set; } = Guid.NewGuid(); public Guid ID { get; private set; } = Guid.NewGuid();
public readonly CIDR Network; public readonly Network4 Network;
public String Name; public String Name;
public DateTime FirstSeen; public DateTime FirstSeen;
@ -29,16 +30,16 @@ namespace ln.skyscanner.entities
public bool AutoScan { get; set; } public bool AutoScan { get; set; }
public IEnumerable<NetworkInterface> AttachedInterfaces => SkyEntities.interfaceCollection.Where((intf) => Network.Contains(intf.IPs.Select((iip)=>iip.IP))); public IEnumerable<NetworkInterface> AttachedInterfaces => SkyEntities.interfaceCollection.Where((intf) => Network.Contains(intf.ConfiguredIPs.Select((iip)=>iip.IP)));
public IEnumerable<Node> AttachedNodes => AttachedInterfaces.Select(intf => intf.Node).Distinct(); public IEnumerable<Node> AttachedNodes => AttachedInterfaces.Select(intf => intf.Node).Distinct();
private Subnet() private Subnet()
{ {
} }
public Subnet(CIDR cidr) public Subnet(Network4 network)
{ {
Network = cidr; Network = network;
Name = Network.ToString(); Name = Network.ToString();
FirstSeen = DateTime.Now; FirstSeen = DateTime.Now;
} }

34
http/CrawlerApi.cs
View File

@ -6,6 +6,7 @@ using ln.skyscanner.crawl;
using ln.types; using ln.types;
using System.Net; using System.Net;
using ln.types.threads; using ln.types.threads;
using ln.types.net;
namespace ln.skyscanner.http namespace ln.skyscanner.http
{ {
@ -36,13 +37,13 @@ namespace ln.skyscanner.http
[Callable] [Callable]
public CrawledHost GetHostByIP(string _ip) public CrawledHost GetHostByIP(string _ip)
{ {
return SkyScanner.Crawler.FindHostForIP(CIDR.Parse(_ip)); return SkyScanner.Crawler.FindHostForIP(IPv4.Parse(_ip));
} }
[Callable] [Callable]
public CrawledHost AddHost(string _ip, string name) public CrawledHost AddHost(string _ip, string name)
{ {
CIDR ip = CIDR.Parse(_ip); IPv4 ip = IPv4.Parse(_ip);
CrawledHost host = SkyScanner.Crawler.FindHostForIP(ip); CrawledHost host = SkyScanner.Crawler.FindHostForIP(ip);
host.Name = name; host.Name = name;
SkyScanner.Crawler.CrawledHosts.Upsert(host); SkyScanner.Crawler.CrawledHosts.Upsert(host);
@ -101,5 +102,34 @@ namespace ln.skyscanner.http
SkyScanner.Crawler.CrawlSubnets = !SkyScanner.Crawler.CrawlSubnets; SkyScanner.Crawler.CrawlSubnets = !SkyScanner.Crawler.CrawlSubnets;
} }
[Callable]
public Network4[] GetBlockedNetworks()
{
return Crawler.BlockedNetworks.ToArray();
}
[Callable]
public void AddBlockedNetwork(string _network)
{
Network4 blocked = Network4.Parse(_network);
if (!Crawler.BlockedNetworks.Contains(blocked))
{
Crawler.BlockedNetworks.Insert(blocked);
}
}
[Callable]
public void RemoveBlockedNetwork(string _network)
{
Network4 blocked = Network4.Parse(_network);
foreach (Network4 blockedNetwork in Crawler.BlockedNetworks)
{
if (blockedNetwork.Equals(blocked))
{
Crawler.BlockedNetworks.Delete(blockedNetwork);
return;
}
}
}
} }
} }

9
http/NetworkApi.cs
View File

@ -12,6 +12,7 @@ using ln.http.resources;
using ln.skyscanner.entities; using ln.skyscanner.entities;
using System.Linq; using System.Linq;
using ln.types; using ln.types;
using ln.types.net;
namespace ln.skyscanner.http namespace ln.skyscanner.http
{ {
public class NetworkApi : JsonCallResource public class NetworkApi : JsonCallResource
@ -39,10 +40,10 @@ namespace ln.skyscanner.http
} }
[Callable] [Callable]
public Node[] GetHostsInSubnet(string _cidr) public Node[] GetHostsInSubnet(string _network)
{ {
CIDR cidr = CIDR.Parse(_cidr); Network4 network = Network4.Parse(_network);
return GlobalNetwork.FindHostsInSubnet(cidr).ToArray(); return GlobalNetwork.FindHostsInSubnet(network).ToArray();
} }
[Callable] [Callable]
@ -52,7 +53,7 @@ namespace ln.skyscanner.http
//foreach (Node node in GlobalNetwork.Nodes) //foreach (Node node in GlobalNetwork.Nodes)
//hopMap.AddNode(node, 0); //hopMap.AddNode(node, 0);
hopMap.AddNode(GlobalNetwork.FindNodeByIP(CIDR.Parse("10.10.10.2")),0); hopMap.AddNode(GlobalNetwork.FindNodeByIP(IPv4.Parse("10.10.10.2")),0);
return hopMap.HopNodes; return hopMap.HopNodes;
} }

2
ln.skyscanner.csproj
View File

@ -43,6 +43,7 @@
<Reference Include="LiteDB"> <Reference Include="LiteDB">
<HintPath>..\packages\LiteDB.4.1.4\lib\net40\LiteDB.dll</HintPath> <HintPath>..\packages\LiteDB.4.1.4\lib\net40\LiteDB.dll</HintPath>
</Reference> </Reference>
<Reference Include="System.Net.Http" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<Compile Include="Program.cs" /> <Compile Include="Program.cs" />
@ -75,6 +76,7 @@
<Compile Include="crawl\service\CrawlService.cs" /> <Compile Include="crawl\service\CrawlService.cs" />
<Compile Include="crawl\service\TCP.cs" /> <Compile Include="crawl\service\TCP.cs" />
<Compile Include="entities\IntfIP.cs" /> <Compile Include="entities\IntfIP.cs" />
<Compile Include="crawl\service\HTTP.cs" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<None Include="packages.config" /> <None Include="packages.config" />

2539
mono_crash.1adb36dd9e.0.json 100644
View File

File diff suppressed because it is too large Load Diff

2
templates/static/crawlerhosts.html
View File

@ -151,7 +151,7 @@
$("#chHints").DataTable({ $("#chHints").DataTable({
columns: [ columns: [
{ title: "Schlüssel", data: "Key" }, { title: "Schlüssel", data: "Key" },
{ title: "Wert", data: "Value" } { title: "Wert", data: "Value", render: function(d,t,r){ return $("<div>").text(d).html(); } }
], ],
}); });

4
templates/static/network/index.html
View File

@ -65,8 +65,8 @@
$("#dInterfaces").empty(); $("#dInterfaces").empty();
$.each( node.Interfaces, function(){ $.each( node.Interfaces, function(){
var intf = this; var intf = this;
$.each( intf.IPs, function(){ $.each( intf.ConfiguredIPs, function(){
$("#dInterfaces").append( $( "<option value='"+ this.IP +"'>" + this.IP + "[ " + intf.Name + " ]</option>" ) ); $("#dInterfaces").append( $( "<option value='"+ this.Network +"'>" + this.IP + "[ " + intf.Name + " ]</option>" ) );
}); });
}); });