Skytron
/
ln.skyscanner
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

WIP

broken
Harald Wolff 2019-04-02 01:25:44 +02:00
parent 4cac9fbf56
commit 5e514b5afc
22 changed files with 2835 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
Program.cs
View File

@ -8,33 +8,10 @@
// *
// **/
using System;
using Renci.SshNet;
using ln.skyscanner.identify;
using System.Net;
using System.Collections.Generic;
using ln.snmp.types;
using System.IO;
using ln.snmp;
using ln.snmp.endpoint;
using System.Diagnostics;
using ln.perfdb.storage;
using ln.logging;
using System.Security.Cryptography;
using ln.snmp.asn1;
using System.Linq;
using ln.types;
using System.Runtime.InteropServices;
using ln.snmp.rfc1213;
using ln.types.sync;
using ln.skyscanner.crawl;
using System.Threading;
using ln.skyscanner.crawl.tests;
using Castle.DynamicProxy;
using ln.types.odb;
using System.Net.Sockets;
using ln.skyscanner.entities;
using ln.types.btree;
using ln.types.test;
using ln.types.threads;
using ln.types.net;
@ -52,7 +29,7 @@ namespace ln.skyscanner
fileLogger.MaxLogLevel = LogLevel.INFO;
Logger.Default.Backends.Add(fileLogger);
Logger.ConsoleLogger.MaxLogLevel = LogLevel.DEBUGDETAIL;
//Logger.ConsoleLogger.MaxLogLevel = LogLevel.DEBUGDETAIL;
Initialize();
@ -60,32 +37,31 @@ namespace ln.skyscanner
skyScanner.Start();
return;
foreach (IntfIP iip in SkyScanner.Instance.Entities.intfIPCollection.Select("Network", Network4.Parse("10.255.9.16/29")))
{
Console.WriteLine("INTF-IP ():{0}", iip);
}
foreach (IntfIP iip in SkyScanner.Instance.Entities.intfIPCollection)
{
Console.WriteLine("INTF-IP: {0}", iip);
}
IEnumerable<Node> nodes = skyScanner.Entities.nodeCollection;
foreach (Node node in nodes)
{
Console.WriteLine("Node: {0}", node);
skyScanner.Entities.nodeCollection.Update(node);
foreach (Node checkNode in SkyScanner.Instance.Entities.nodeCollection.Select("PrimaryIP", node.PrimaryIP))
{
Console.WriteLine("Node (lookup primary ip): {0}", checkNode);
if (!checkNode.ID.Equals(node))
{
if ((checkNode.Interfaces.Count() == 0))
{
SkyScanner.Instance.Entities.nodeCollection.Delete(checkNode);
}
}
}
Timing.Meassure("", () =>
{
foreach (Node neighbor in SkyScanner.Instance.Entities.GlobalNetwork.FindNeighbors(node))
{
Console.WriteLine(" Neighbor: {0}", neighbor);
}
});
//IEnumerable<NetworkInterface> interfaces = Timing.Meassure("node.Interfaces",() => node.Interfaces);
//foreach (NetworkInterface intf in interfaces)
//{
// Console.WriteLine(" Interface: {0}", intf);
//}
Console.WriteLine("");
Console.WriteLine("");
Console.WriteLine("");
}

1
SkyEntities.cs
View File

@ -40,6 +40,7 @@ namespace ln.skyscanner
nodeCollection.EnsureIndex("PrimaryIP");
interfaceCollection.EnsureIndex("NodeID");
intfIPCollection.EnsureIndex("interfaceID");
intfIPCollection.EnsureIndex("Network");
GlobalNetwork = new GlobalNetwork(this);
}

4
crawl/Crawl.cs
View File

@ -100,10 +100,8 @@ namespace ln.skyscanner.crawl
try
{
while (servicesToCheck.Count > 0)
RunServiceCheck(servicesToCheck[0]);
} catch (Exception e)
{
Logging.Log(LogLevel.ERROR, "Crawl.RunJob(): Caught exception: {0}",e);
@ -112,7 +110,7 @@ namespace ln.skyscanner.crawl
Host.LastCheckTime = DateTime.Now - dateTime;
Host.LastCheck = dateTime;
Host.NextCheck = dateTime + TimeSpan.FromHours(1);
Host.NextCheck = dateTime + TimeSpan.FromHours(24);
bool updated = Crawler.CrawledHosts.Upsert( Host );

11
crawl/CrawlNetwork.cs
View File

@ -16,6 +16,7 @@ using ln.logging;
using Newtonsoft.Json;
using System.Linq;
using ln.types.odb;
using ln.types.net;
namespace ln.skyscanner.crawl
{
public class CrawlNetwork : PoolJob, IPersistent
@ -24,7 +25,7 @@ namespace ln.skyscanner.crawl
public Crawler Crawler { get; }
public CrawledSubnet Subnet { get; }
public CIDR Network { get; }
public Network4 Network { get; }
@ -36,7 +37,7 @@ namespace ln.skyscanner.crawl
Name = String.Format("Subnet crawl {0}", subnet.Network);
}
public CrawlNetwork(Crawler crawler, CrawledSubnet subnet,CIDR network)
public CrawlNetwork(Crawler crawler, CrawledSubnet subnet,Network4 network)
{
Crawler = crawler;
Subnet = subnet;
@ -47,9 +48,9 @@ namespace ln.skyscanner.crawl
public override void RunJob()
{
if (Network.MaskWidth < 26)
if (Network.Width < 26)
{
SplitJob(Network.Divide(26 - Network.MaskWidth).Select((n) => new CrawlNetwork(Crawler, Subnet, n)).ToArray());
SplitJob(Network.Divide(26 - (int)Network.Width).Select((n) => new CrawlNetwork(Crawler, Subnet, n)).ToArray());
return;
}
@ -62,7 +63,7 @@ namespace ln.skyscanner.crawl
int count = 0;
foreach (CIDR ip in Network)
foreach (IPv4 ip in Network)
{
count++;

16
crawl/CrawledHost.cs
View File

@ -15,6 +15,7 @@ using System.Linq;
using ln.snmp.rfc1213;
using LiteDB;
using ln.types.odb;
using ln.types.net;
namespace ln.skyscanner.crawl
{
@ -24,13 +25,14 @@ namespace ln.skyscanner.crawl
public readonly Guid ID = Guid.NewGuid();
public String Name { get; set; }
public CIDR PrimaryIP { get; set; }
public IPv4 PrimaryIP { get; set; }
public String[] HWAddresses = new string[0];
public String PrimaryHWAddr => HWAddresses.Where((hwa) => hwa != null && !String.Empty.Equals(hwa)).FirstOrDefault();
public CIDR[] IPAddresses { get; set; } = new CIDR[0];
public IPv4[] IPAddresses { get; set; } = new IPv4[0];
public Network4[] Networks { get; set; } = new Network4[0];
Dictionary<string, object> hints = new Dictionary<string, object>();
public DateTime FirstSeen { get; set; }
@ -46,13 +48,13 @@ namespace ln.skyscanner.crawl
public CrawledHost()
{
Name = "noname";
PrimaryIP = IPAddress.Any;
PrimaryIP = IPv4.ANY;
}
public bool HasIP(CIDR ip)
public bool HasIP(IPv4 ip)
{
foreach (CIDR _ip in IPAddresses)
if (_ip.Host.Equals(ip))
foreach (IPv4 _ip in IPAddresses)
if (_ip.Equals(ip))
return true;
return false;
}

7
crawl/CrawledSubnet.cs
View File

@ -12,6 +12,7 @@ using ln.types;
using ln.types.odb;
using System.Collections.Generic;
using System.Linq;
using ln.types.net;
namespace ln.skyscanner.crawl
{
public class CrawledSubnet
@ -19,7 +20,7 @@ namespace ln.skyscanner.crawl
[DocumentID]
Guid ID = Guid.NewGuid();
public readonly CIDR Network;
public readonly Network4 Network;
public String Name;
public DateTime FirstSeen;
@ -33,9 +34,9 @@ namespace ln.skyscanner.crawl
FirstSeen = DateTime.Now;
}
public CrawledSubnet(CIDR cidr)
public CrawledSubnet(Network4 network)
{
Network = cidr;
Network = network;
Name = Network.ToString();
FirstSeen = DateTime.Now;
}

15
crawl/Crawler.cs
View File

@ -31,6 +31,7 @@ using ln.types.odb;
using ln.skyscanner.crawl.service;
using ln.skyscanner.crawl.tests;
using LiteDB;
using ln.types.net;
namespace ln.skyscanner.crawl
{
@ -41,6 +42,7 @@ namespace ln.skyscanner.crawl
CrawlService.RegisterService(new TCP(new int[] { 13080, 13022, 80, 22, 443, 13443 }));
CrawlService.RegisterService(new SNMP(new string[] { "VhclfC7lfIojYZ", "Vhclf(C7$lfIojYZ", "ByFR4oW98hap", "qVy3hnZJ2fov" }));
CrawlService.RegisterService(new RFC1213());
CrawlService.RegisterService(new HTTP());
}
public SkyScanner SkyScanner { get; }
@ -67,12 +69,12 @@ namespace ln.skyscanner.crawl
public ODBCollection<CrawledHost> CrawledHosts => hosts;
public ODBCollection<CrawledSubnet> CrawledSubnets => subnets;
//private ODB<CrawlPool> dbCrawlPool;
public ODBCollection<Network4> BlockedNetworks => blockedNetworks;
private ODB odbDatabase;
private ODBCollection<CrawledHost> hosts;
private ODBCollection<CrawledSubnet> subnets;
private ODBCollection<Network4> blockedNetworks;
public Crawler(SkyScanner skyScanner)
@ -94,6 +96,7 @@ namespace ln.skyscanner.crawl
odbDatabase = new ODB(BasePath);
hosts = odbDatabase.GetCollection<CrawledHost>();
subnets = odbDatabase.GetCollection<CrawledSubnet>();
blockedNetworks = odbDatabase.GetCollection<Network4>("blockedNetworks");
CrawlerState = ComponentState.INITIALIZED;
}
@ -168,7 +171,7 @@ namespace ln.skyscanner.crawl
}
}
public void EnsureSubnet(CIDR network)
public void EnsureSubnet(Network4 network)
{
FindSubnet(network);
}
@ -195,7 +198,7 @@ namespace ln.skyscanner.crawl
crawlThreadPool.Enqueue(crawlSubnet);
}
public CrawledHost FindHostForIP(CIDR ip)
public CrawledHost FindHostForIP(IPv4 ip)
{
CrawledHost crawledHost = CrawledHosts.Where(host => host.HasIP(ip)).FirstOrDefault();
if (crawledHost == null)
@ -208,7 +211,7 @@ namespace ln.skyscanner.crawl
}
return crawledHost;
}
public CrawledSubnet FindSubnet(CIDR network)
public CrawledSubnet FindSubnet(Network4 network)
{
CrawledSubnet sn = CrawledSubnets.Where(subnet => subnet.Network.Equals(network)).FirstOrDefault();
if (sn == null)
@ -238,7 +241,7 @@ namespace ln.skyscanner.crawl
}
if (CrawlSubnets)
foreach (CrawledSubnet subnet in CrawledSubnets.Where( sn => (sn.NextScan < (DateTime.Now - TimeSpan.FromDays(1)) && sn.Network.MaskWidth >= 24)))
foreach (CrawledSubnet subnet in CrawledSubnets.Where( sn => (sn.NextScan < DateTime.Now ) && sn.Network.Width >= 24))
{
Crawl(subnet);
}

92
crawl/service/HTTP.cs 100644
View File

@ -0,0 +1,92 @@
// /**
// * File: HTTP.cs
// * Author: haraldwolff
// *
// * This file and it's content is copyrighted by the Author and / or copyright holder.
// * Any use wihtout proper permission is illegal and may lead to legal actions.
// *
// *
// **/
using System;
using System.Net.Http;
using ln.logging;
using ln.types.net;
using System.Linq;
using System.Net;
using System.IO;
namespace ln.skyscanner.crawl.service
{
public class HTTP : CrawlService
{
public HTTP()
:base("http")
{
}
public override bool Check(Crawl crawl)
{
foreach (int port in new int[] { 13080, 80 })
{
if (crawl.CheckRequiredOption("tcp", port))
{
if (CheckPort(crawl, port))
return true;
}
}
return false;
}
private bool CheckPort(Crawl crawl,int port)
{
foreach (IPv4 ip in crawl.Host.IPAddresses.Concat(new IPv4[] { crawl.Host.PrimaryIP }).Distinct())
{
try
{
crawl.setState("Check HTTP at {0}:{1}",ip,port);
HttpWebRequest webRequest = WebRequest.CreateHttp(String.Format("http://{0}:{1}", ip, port));
webRequest.AllowAutoRedirect = true;
HttpWebResponse response = (HttpWebResponse)webRequest.GetResponse();
String content = null;
using (StreamReader reader = new StreamReader(response.GetResponseStream()))
{
content = reader.ReadToEnd();
reader.Close();
}
crawl.Host.SetHint("http.ip", ip);
crawl.Host.SetHint("http.port", port);
crawl.Host.SetHint("http.status", response.StatusCode);
crawl.Host.SetHint("http.server", response.Server);
crawl.Host.SetHint("http.content", content);
return true;
}
catch (WebException e)
{
Logging.Log(e);
}
}
return false;
}
public override bool HostProvidesOption(Crawl crawl, params object[] parameters)
{
if (parameters.Length == 0)
return false;
foreach (object oport in parameters)
{
int port = (int)oport;
if (!crawl.Host.GetHint(String.Format("http.{0}", port), false))
return false;
}
return true;
}
}
}

7
crawl/service/RFC1213.cs
View File

@ -49,7 +49,12 @@ namespace ln.skyscanner.crawl.tests
crawl.Host.IPAddresses = interfaces.SelectMany(intf => intf.IPAddresses).ToArray();
crawl.Host.HWAddresses = interfaces.Select(intf => intf.HWAddr).ToArray();
crawl.Host.SetHint("rfc1213.interfaces",interfaces.Select(intf => string.Format("{0};{1};{2}",intf.Name,intf.HWAddr,String.Join(",",intf.IPAddresses.Select(ip=>ip.ToString())))).ToArray());
crawl.Host.SetHint("rfc1213.interfaces", interfaces.Select(intf => string.Format("{0};{1};{2}",
intf.Name,
intf.HWAddr,
String.Join(",", intf.IPAddresses.Zip(intf.Netmasks,(ip,mask) => String.Format("{0}/{1}",ip.ToString(),mask.ToString())))
)).ToArray()
);
crawl.Host.SetHint("rfc1213", true);
}
} catch (TimeoutException)

17
crawl/service/SNMP.cs
View File

@ -16,6 +16,7 @@ using ln.snmp.types;
using System.Runtime.Remoting.Messaging;
using ln.skyscanner.crawl.service;
using System.Linq;
using ln.types.net;
namespace ln.skyscanner.crawl.tests
{
public class SNMP : CrawlService
@ -71,9 +72,9 @@ namespace ln.skyscanner.crawl.tests
private bool TestSnmpV3(CrawledHost crawledHost)
{
CIDR[] ips = crawledHost.IPAddresses.Concat(new CIDR[] { crawledHost.PrimaryIP }).Distinct().ToArray();
IPv4[] ips = crawledHost.IPAddresses.Concat(new IPv4[] { crawledHost.PrimaryIP }).Distinct().ToArray();
foreach (CIDR ip in ips)
foreach (IPv4 ip in ips)
{
using (USMEndpoint v3endpoint = new USMEndpoint(SNMPEngine.DefaultEngine, new IPEndPoint(ip, 161)))
{
@ -89,7 +90,6 @@ namespace ln.skyscanner.crawl.tests
{
crawledHost.SetHint("snmp.version", 3);
bool replied = false;
int c = 0;
foreach (string community in defaultCommunities)
@ -114,6 +114,7 @@ namespace ln.skyscanner.crawl.tests
}
catch (TimeoutException)
{
return false;
}
catch (SnmpError)
{
@ -128,9 +129,9 @@ namespace ln.skyscanner.crawl.tests
private bool TestSnmpV2(CrawledHost crawledHost)
{
CIDR[] ips = crawledHost.IPAddresses.Concat(new CIDR[] { crawledHost.PrimaryIP }).Distinct().ToArray();
IPv4[] ips = crawledHost.IPAddresses.Concat(new IPv4[] { crawledHost.PrimaryIP }).Distinct().ToArray();
foreach (CIDR ip in ips)
foreach (IPv4 ip in ips)
{
using (SnmpV2Endpoint v2endpoint = new SnmpV2Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(crawledHost.PrimaryIP, 161)))
{
@ -162,9 +163,9 @@ namespace ln.skyscanner.crawl.tests
private bool TestSnmpV1(CrawledHost crawledHost)
{
CIDR[] ips = crawledHost.IPAddresses.Concat(new CIDR[] { crawledHost.PrimaryIP }).Distinct().ToArray();
IPv4[] ips = crawledHost.IPAddresses.Concat(new IPv4[] { crawledHost.PrimaryIP }).Distinct().ToArray();
foreach (CIDR ip in ips)
foreach (IPv4 ip in ips)
{
using (SnmpV1Endpoint v1endpoint = new SnmpV1Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(crawledHost.PrimaryIP, 161)))
{
@ -237,7 +238,7 @@ namespace ln.skyscanner.crawl.tests
public static SnmpInterface GetSnmpInterface(CrawledHost crawledHost)
{
int snmpVersion = crawledHost.GetHint<int>("snmp.version", -1);
CIDR snmpAddress = crawledHost.GetHint<CIDR>("snmp.address", crawledHost.PrimaryIP);
IPv4 snmpAddress = crawledHost.GetHint<IPv4>("snmp.address", crawledHost.PrimaryIP);
switch (snmpVersion)
{

11
crawl/service/SSH.cs
View File

@ -12,6 +12,7 @@ using Renci.SshNet;
using ln.types;
using Renci.SshNet.Common;
using System.Net.Sockets;
using ln.types.net;
namespace ln.skyscanner.crawl.tests
{
public static class SSH
@ -37,11 +38,11 @@ namespace ln.skyscanner.crawl.tests
public static bool CanConnect(CrawledHost crawledHost)
{
int sshPort = crawledHost.GetHint<int>("ssh.port", -1);
CIDR sshIP = crawledHost.GetHint<CIDR>("ssh.ip", null);
IPv4 sshIP = crawledHost.GetHint<IPv4>("ssh.ip", null);
string sshUser = crawledHost.GetHint<string>("ssh.login", null);
string sshPassword = crawledHost.GetHint<string>("ssh.password", null);
if ((sshPort == -1) || !CanConnect(crawledHost,sshIP.Host,sshPort,sshUser,sshPassword))
if ((sshPort == -1) || !CanConnect(crawledHost,sshIP,sshPort,sshUser,sshPassword))
{
if (!Scan(crawledHost))
{
@ -58,7 +59,7 @@ namespace ln.skyscanner.crawl.tests
private static bool Scan(CrawledHost crawledHost)
{
foreach (CIDR ip in crawledHost.IPAddresses)
foreach (IPv4 ip in crawledHost.IPAddresses)
{
foreach (int port in new int[] { 13022, 22 })
{
@ -66,7 +67,7 @@ namespace ln.skyscanner.crawl.tests
{
foreach (string password in defaultPasswords)
{
if (CanConnect(crawledHost, ip.Host, port, "skytron", password, true))
if (CanConnect(crawledHost, ip, port, "skytron", password, true))
return true;
}
} catch (SocketException)
@ -78,7 +79,7 @@ namespace ln.skyscanner.crawl.tests
return false;
}
private static bool CanConnect(CrawledHost crawledHost, CIDR host, int port, string username, string password,bool throwe = false)
private static bool CanConnect(CrawledHost crawledHost, IPv4 host, int port, string username, string password,bool throwe = false)
{
using (TcpClient tcp = new TcpClient())
{

104
entities/GlobalNetwork.cs
View File

@ -40,7 +40,7 @@ namespace ln.skyscanner.entities
SkyEntities = skyEntities;
}
public void EnsureSubnet(CIDR network)
public void EnsureSubnet(Network4 network)
{
SkyScanner.Crawler.EnsureSubnet(network);
@ -58,35 +58,39 @@ namespace ln.skyscanner.entities
return SkyEntities.nodeCollection.Select(id);
}
public Subnet FindSubnetForHost(CIDR host)
public Subnet FindSubnetForIP(IPv4 ip)
{
return SkyEntities.subnetCollection.Select(net => net.Network.Contains(host)).FirstOrDefault();
return SkyEntities.subnetCollection.Select(net => net.Network.Contains(ip)).FirstOrDefault();
}
public IEnumerable<Node> FindHostsInSubnet(CIDR network)
public IEnumerable<Node> FindHostsInSubnet(Network4 network)
{
IntfIP[] intfIPs = SkyEntities.intfIPCollection.Select((ip) => ip.IP.In(network)).ToArray();
NetworkInterface[] networkInterfaces = intfIPs.Select((iip) => iip.NetworkInterface).Where((i)=>i!=null).Distinct().ToArray();
IntfIP[] intfIPs = SkyEntities.intfIPCollection.Select("Network",network).ToArray();
NetworkInterface[] networkInterfaces = intfIPs.Select((iip) => iip.NetworkInterface).ToArray();
Node[] nodes = SkyEntities.nodeCollection.Select(networkInterfaces.Select((ni) => new ODBGuid(ni.NodeID))).ToArray();
return nodes;
}
public IEnumerable<Node> FindNeighbors(Node node)
{
HashSet<Node> nodes = new HashSet<Node>();
foreach (Node neighbor in node.IPAdresses.SelectMany(
(ip) => Timing.Meassure("FindHostsInSubnet", () =>
FindHostsInSubnet(ip)
)
))
nodes.Add(neighbor);
foreach (Network4 network in node.Networks)
{
foreach (Node neighbor in FindHostsInSubnet(network))
{
nodes.Add(neighbor);
}
}
return nodes;
}
public Node FindNodeByIP(IEnumerable<CIDR> ips)
public Node FindNodeByIP(IEnumerable<IPv4> ips)
{
foreach (CIDR ip in ips)
foreach (IPv4 ip in ips)
{
Node node = FindNodeByIP(ip);
if (node != null)
@ -94,17 +98,17 @@ namespace ln.skyscanner.entities
}
return null;
}
public Node FindNodeByIP(CIDR ip)
{
ip = ip.Host;
/*
* ToDo: Alternative Implementation
* a) higher performance (use indexing)
* b) fix bug: Node not found if no interface ips exist...
*/
IntfIP _iip = SkyEntities.intfIPCollection.Where((iip) => iip.IP.Host.Equals(ip)).FirstOrDefault();
return _iip?.NetworkInterface?.Node;
}
//public Node FindNodeByIP(CIDR ip)
//{
// ip = ip.Host;
// /*
// * ToDo: Alternative Implementation
// * a) higher performance (use indexing)
// * b) fix bug: Node not found if no interface ips exist...
// */
// IntfIP _iip = SkyEntities.intfIPCollection.Where((iip) => iip.IP.Host.Equals(ip)).FirstOrDefault();
// return _iip?.NetworkInterface?.Node;
//}
public Node FindNodeByIP(IPv4 ip)
{
@ -196,30 +200,29 @@ namespace ln.skyscanner.entities
SkyEntities.interfaceCollection.Upsert(networkInterface);
CIDR[] crawledIPs = fields[2].Split(',').Where((sip) => !String.Empty.Equals(sip)).Select((sip) => CIDR.Parse(sip)).ToArray();
IntfIP[] currentIPs = networkInterface.IPs.ToArray();
IPv4[][] crawledIPs = fields[2].Split(',').Where((sip) => !String.Empty.Equals(sip))
.Select((sip) => sip.Split('/').Select(ip => IPv4.Parse(ip)).ToArray()).ToArray();
foreach (CIDR crawledIP in crawledIPs)
IntfIP[] currentIPs = networkInterface.ConfiguredIPs.ToArray();
foreach (IPv4[] crawledIP in crawledIPs)
{
int n;
for (n=0; n < currentIPs.Length; n++)
if (currentIPs[n].IP.Equals(crawledIP))
if (currentIPs[n].IP.Equals(crawledIP[0]))
break;
if (n == currentIPs.Length)
{
IntfIP miss = new IntfIP(networkInterface, crawledIP);
IntfIP miss = new IntfIP(networkInterface, crawledIP[0], new Network4(crawledIP[0], crawledIP[1]));
SkyEntities.intfIPCollection.Upsert(miss);
}
}
foreach (IntfIP iip in currentIPs)
{
CIDR ip = iip.IP;
IPv4 ip = iip.IP;
if (crawledIPs.Contains(ip))
{
}
else
if (!crawledIPs.Select((ipl)=>ipl[0]).Contains(ip))
{
SkyEntities.intfIPCollection.Delete(iip);
}
@ -234,7 +237,7 @@ namespace ln.skyscanner.entities
String.Format("ssh://{0}:{1}@{2}:{3}",
crawledHost.GetHint<string>("ssh.login"),
crawledHost.GetHint<string>("ssh.password"),
crawledHost.GetHint<CIDR>("ssh.ip"),
crawledHost.GetHint<IPv4>("ssh.ip"),
crawledHost.GetHint<int>("ssh.port")
)
)
@ -242,11 +245,36 @@ namespace ln.skyscanner.entities
}
foreach (CIDR ip in node.IPAdresses)
node.RemoveURI("http");
if (crawledHost.GetHint("http.port",-1) != -1)
{
EnsureSubnet(ip.Network);
node.AddURI(new URI(String.Format("http://{0}:{1}",
crawledHost.GetHint<IPv4>("http.ip", null),
crawledHost.GetHint<int>("http.port"))
));
}
string httpIndex = crawledHost.GetHint("http.content", "");
if (httpIndex.Contains("<title>RouterOS router configuration page</title>"))
{
node.Vendor = "MicroTik";
}
if (crawledHost.GetHint("snmp.orids",new string[0]).Contains("1.3.6.1.4.1.41112"))
{
node.Vendor = "Ubiquity";
}
foreach (Network4 network in node.Networks)
EnsureSubnet(network);
node.LastUpdate = DateTime.Now;
SkyEntities.nodeCollection.Upsert(node);
}

12
entities/IntfIP.cs
View File

@ -11,6 +11,7 @@ using System;
using ln.types.odb;
using ln.types;
using Newtonsoft.Json;
using ln.types.net;
namespace ln.skyscanner.entities
{
public class IntfIP
@ -23,18 +24,23 @@ namespace ln.skyscanner.entities
[JsonIgnore]
public NetworkInterface NetworkInterface => SkyScanner.Instance.Entities.interfaceCollection.Select(interfaceID);
public CIDR IP { get; private set; }
public IPv4 IP { get; private set; }
public Network4 Network { get; private set; }
public IntfIP()
{
}
public IntfIP(NetworkInterface networkInterface,CIDR ip)
public IntfIP(NetworkInterface networkInterface,IPv4 ip,Network4 network)
{
interfaceID = networkInterface.ID;
IP = ip;
Network = network;
}
public override string ToString()
{
return string.Format("[IntfIP ID={0} IP={1,18} Network={2}]", ID, IP, Network);
}
}

12
entities/NetworkInterface.cs
View File

@ -15,6 +15,7 @@ using ln.types.odb;
using Newtonsoft.Json;
using LiteDB;
using ln.logging;
using ln.types.net;
namespace ln.skyscanner.entities
{
public class NetworkInterface
@ -34,8 +35,7 @@ namespace ln.skyscanner.entities
public string Name { get; private set; } = "";
public string HWAddress { get; set; } = "";
//public CIDR[] IPs { get; set; } = new CIDR[0];
public IEnumerable<IntfIP> IPs => SkyScanner.Instance.Entities.intfIPCollection.Select("interfaceID", ID);
public IEnumerable<IntfIP> ConfiguredIPs => SkyScanner.Instance.Entities.intfIPCollection.Select("interfaceID", ID);
private NetworkInterface()
{ }
@ -46,13 +46,11 @@ namespace ln.skyscanner.entities
Name = name;
}
public bool HasIP(CIDR ip)
public bool HasIP(IPv4 ip)
{
ip = ip.Host;
foreach (IntfIP ifip in IPs)
foreach (IntfIP ifip in ConfiguredIPs)
{
if (ifip.IP.Host.Equals(ip))
if (ifip.IP.Equals(ip))
return true;
}
return false;

8
entities/Node.cs
View File

@ -39,8 +39,12 @@ namespace ln.skyscanner.entities
public string ProductLine { get; set; }
[JsonIgnore]
public IEnumerable<NetworkInterface> Interfaces => SkyScanner.Instance.Entities.interfaceCollection.Select("NodeID", ID);
public IEnumerable<CIDR> IPAdresses => Interfaces.SelectMany(intf => intf.IPs).Select((iip) => iip.IP).Distinct();
[JsonIgnore]
public IEnumerable<IPv4> IPAdresses => Interfaces.SelectMany(intf => intf.ConfiguredIPs.Select(nip => nip.IP));
[JsonIgnore]
public IEnumerable<Network4> Networks => Interfaces.SelectMany(intf => intf.ConfiguredIPs.Select(nip => nip.Network));
private HashSet<URI> uris = new HashSet<URI>();
@ -48,7 +52,7 @@ namespace ln.skyscanner.entities
{
}
public Node(CIDR primaryIP)
public Node(IPv4 primaryIP)
{
PrimaryIP = primaryIP;
Name = PrimaryIP.ToString();

9
entities/Subnet.cs
View File

@ -12,6 +12,7 @@ using ln.types;
using ln.types.odb;
using System.Collections.Generic;
using System.Linq;
using ln.types.net;
namespace ln.skyscanner.entities
{
public class Subnet : Persistent
@ -21,7 +22,7 @@ namespace ln.skyscanner.entities
[DocumentID]
public Guid ID { get; private set; } = Guid.NewGuid();
public readonly CIDR Network;
public readonly Network4 Network;
public String Name;
public DateTime FirstSeen;
@ -29,16 +30,16 @@ namespace ln.skyscanner.entities
public bool AutoScan { get; set; }
public IEnumerable<NetworkInterface> AttachedInterfaces => SkyEntities.interfaceCollection.Where((intf) => Network.Contains(intf.IPs.Select((iip)=>iip.IP)));
public IEnumerable<NetworkInterface> AttachedInterfaces => SkyEntities.interfaceCollection.Where((intf) => Network.Contains(intf.ConfiguredIPs.Select((iip)=>iip.IP)));
public IEnumerable<Node> AttachedNodes => AttachedInterfaces.Select(intf => intf.Node).Distinct();
private Subnet()
{
}
public Subnet(CIDR cidr)
public Subnet(Network4 network)
{
Network = cidr;
Network = network;
Name = Network.ToString();
FirstSeen = DateTime.Now;
}

34
http/CrawlerApi.cs
View File

@ -6,6 +6,7 @@ using ln.skyscanner.crawl;
using ln.types;
using System.Net;
using ln.types.threads;
using ln.types.net;
namespace ln.skyscanner.http
{
@ -36,13 +37,13 @@ namespace ln.skyscanner.http
[Callable]
public CrawledHost GetHostByIP(string _ip)
{
return SkyScanner.Crawler.FindHostForIP(CIDR.Parse(_ip));
return SkyScanner.Crawler.FindHostForIP(IPv4.Parse(_ip));
}
[Callable]
public CrawledHost AddHost(string _ip, string name)
{
CIDR ip = CIDR.Parse(_ip);
IPv4 ip = IPv4.Parse(_ip);
CrawledHost host = SkyScanner.Crawler.FindHostForIP(ip);
host.Name = name;
SkyScanner.Crawler.CrawledHosts.Upsert(host);
@ -101,5 +102,34 @@ namespace ln.skyscanner.http
SkyScanner.Crawler.CrawlSubnets = !SkyScanner.Crawler.CrawlSubnets;
}
[Callable]
public Network4[] GetBlockedNetworks()
{
return Crawler.BlockedNetworks.ToArray();
}
[Callable]
public void AddBlockedNetwork(string _network)
{
Network4 blocked = Network4.Parse(_network);
if (!Crawler.BlockedNetworks.Contains(blocked))
{
Crawler.BlockedNetworks.Insert(blocked);
}
}
[Callable]
public void RemoveBlockedNetwork(string _network)
{
Network4 blocked = Network4.Parse(_network);
foreach (Network4 blockedNetwork in Crawler.BlockedNetworks)
{
if (blockedNetwork.Equals(blocked))
{
Crawler.BlockedNetworks.Delete(blockedNetwork);
return;
}
}
}
}
}

9
http/NetworkApi.cs
View File

@ -12,6 +12,7 @@ using ln.http.resources;
using ln.skyscanner.entities;
using System.Linq;
using ln.types;
using ln.types.net;
namespace ln.skyscanner.http
{
public class NetworkApi : JsonCallResource
@ -39,10 +40,10 @@ namespace ln.skyscanner.http
}
[Callable]
public Node[] GetHostsInSubnet(string _cidr)
public Node[] GetHostsInSubnet(string _network)
{
CIDR cidr = CIDR.Parse(_cidr);
return GlobalNetwork.FindHostsInSubnet(cidr).ToArray();
Network4 network = Network4.Parse(_network);
return GlobalNetwork.FindHostsInSubnet(network).ToArray();
}
[Callable]
@ -52,7 +53,7 @@ namespace ln.skyscanner.http
//foreach (Node node in GlobalNetwork.Nodes)
//hopMap.AddNode(node, 0);
hopMap.AddNode(GlobalNetwork.FindNodeByIP(CIDR.Parse("10.10.10.2")),0);
hopMap.AddNode(GlobalNetwork.FindNodeByIP(IPv4.Parse("10.10.10.2")),0);
return hopMap.HopNodes;
}

2
ln.skyscanner.csproj
View File

@ -43,6 +43,7 @@
<Reference Include="LiteDB">
<HintPath>..\packages\LiteDB.4.1.4\lib\net40\LiteDB.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http" />
</ItemGroup>
<ItemGroup>
<Compile Include="Program.cs" />
@ -75,6 +76,7 @@
<Compile Include="crawl\service\CrawlService.cs" />
<Compile Include="crawl\service\TCP.cs" />
<Compile Include="entities\IntfIP.cs" />
<Compile Include="crawl\service\HTTP.cs" />
</ItemGroup>
<ItemGroup>
<None Include="packages.config" />

2539
mono_crash.1adb36dd9e.0.json 100644
View File

File diff suppressed because it is too large Load Diff

2
templates/static/crawlerhosts.html
View File

@ -151,7 +151,7 @@
$("#chHints").DataTable({
columns: [
{ title: "Schlüssel", data: "Key" },
{ title: "Wert", data: "Value" }
{ title: "Wert", data: "Value", render: function(d,t,r){ return $("<div>").text(d).html(); } }
],
});

4
templates/static/network/index.html
View File

@ -65,8 +65,8 @@
$("#dInterfaces").empty();
$.each( node.Interfaces, function(){
var intf = this;
$.each( intf.IPs, function(){
$("#dInterfaces").append( $( "<option value='"+ this.IP +"'>" + this.IP + "[ " + intf.Name + " ]</option>" ) );
$.each( intf.ConfiguredIPs, function(){
$("#dInterfaces").append( $( "<option value='"+ this.Network +"'>" + this.IP + "[ " + intf.Name + " ]</option>" ) );
});
});