Skytron
/
ln.skyscanner
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

WIP

broken
Harald Wolff 2019-04-01 07:48:27 +02:00
parent f8d235309e
commit b1e7d86aea
4 changed files with 65 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
crawl/Crawl.cs
View File

@ -98,19 +98,22 @@ namespace ln.skyscanner.crawl
DateTime dateTime = DateTime.Now;
while (servicesToCheck.Count > 0)
RunServiceCheck(servicesToCheck[0]);
try
{
//if (CrawledHost.FirstSeen.Equals(DateTime.MinValue))
// CrawledHost.FirstSeen = DateTime.Now;
while (servicesToCheck.Count > 0)
RunServiceCheck(servicesToCheck[0]);
////CrawledHost.LastSeen = DateTime.Now;
Host.LastCheckTime = DateTime.Now - dateTime;
Host.LastCheck = dateTime;
Host.NextCheck = dateTime + TimeSpan.FromHours(1);
} catch (Exception e)
{
Logging.Log(LogLevel.ERROR, "Crawl.RunJob(): Caught exception: {0}",e);
Logging.Log(e);
}
Host.LastCheckTime = DateTime.Now - dateTime;
Host.LastCheck = dateTime;
Host.NextCheck = dateTime + TimeSpan.FromHours(1);
bool updated = Crawler.CrawledHosts.Update( Host );
bool updated = Crawler.CrawledHosts.Upsert( Host );
setState("Updating global network");
SkyScanner.Instance.Entities.GlobalNetwork.EnqueueUpdate(Host);

52
entities/GlobalNetwork.cs
View File

@ -65,14 +65,22 @@ namespace ln.skyscanner.entities
public IEnumerable<Node> FindHostsInSubnet(CIDR network)
{
IntfIP[] intfIPs = SkyEntities.intfIPCollection.Select((ip) => ip.IP.In(network)).ToArray();
NetworkInterface[] networkInterfaces = intfIPs.Select((iip) => iip.NetworkInterface).Distinct().ToArray();
Node[] nodes = SkyEntities.nodeCollection.Select(networkInterfaces.Select((ni) => (ODBValue)ni.NodeID).Distinct()).ToArray();
NetworkInterface[] networkInterfaces = intfIPs.Select((iip) => iip.NetworkInterface).Where((i)=>i!=null).Distinct().ToArray();
Node[] nodes = SkyEntities.nodeCollection.Select(networkInterfaces.Select((ni) => new ODBGuid(ni.NodeID))).ToArray();
return nodes;
}
public IEnumerable<Node> FindNeighbors(Node node)
{
return node.IPAdresses.SelectMany((ip) => Timing.Meassure("FindHostsInSubnet", () => FindHostsInSubnet(ip)).Distinct().ToArray());
HashSet<Node> nodes = new HashSet<Node>();
foreach (Node neighbor in node.IPAdresses.SelectMany(
(ip) => Timing.Meassure("FindHostsInSubnet", () =>
FindHostsInSubnet(ip)
)
))
nodes.Add(neighbor);
return nodes;
}
public Node FindNodeByIP(IEnumerable<CIDR> ips)
@ -114,16 +122,23 @@ namespace ln.skyscanner.entities
}
while (true)
{
CrawledHost crawledHost = null;
lock (updateQueue)
try
{
if (updateQueue.Count > 0)
crawledHost = updateQueue.Dequeue();
else
break;
}
CrawledHost crawledHost = null;
lock (updateQueue)
{
if (updateQueue.Count > 0)
crawledHost = updateQueue.Dequeue();
else
break;
}
Update(crawledHost);
Update(crawledHost);
} catch (Exception e)
{
Logging.Log(LogLevel.ERROR, "GlobalNetwork.Update(): Caught Exception: {0}",e);
Logging.Log(e);
}
}
//SkyScanner.Entities.DBGlobalNetwork.SavePersistent(SkyScanner.Entities.DBGlobalNetwork.Root);
@ -142,8 +157,17 @@ namespace ln.skyscanner.entities
node = new Node(crawledHost.PrimaryIP);
node.Name = crawledHost.Name;
node.PrimaryMac = crawledHost.PrimaryHWAddr;
SkyEntities.nodeCollection.Insert(node);
}
if (!crawledHost.PrimaryIP.Equals(node.PrimaryIP))
{
Logging.Log(LogLevel.INFO,"GlobalNetwork.Update(): will not update Node with unmatched PrimaryIP {0} != {1}",node.PrimaryIP, crawledHost.PrimaryIP);
return;
}
foreach (String si in crawledHost.GetHint<string[]>("rfc1213.interfaces",new string[0]))
{
String[] fields = si.Split(';');
@ -178,7 +202,6 @@ namespace ln.skyscanner.entities
if (crawledIPs.Contains(ip))
{
EnsureSubnet(ip.Network);
}
else
{
@ -203,6 +226,11 @@ namespace ln.skyscanner.entities
}
foreach (CIDR ip in node.IPAdresses)
{
EnsureSubnet(ip.Network);
}
node.LastUpdate = DateTime.Now;
SkyEntities.nodeCollection.Upsert(node);
}

10
entities/Node.cs
View File

@ -92,5 +92,15 @@ namespace ln.skyscanner.entities
return GetInterface(intfName) != null;
}
public override int GetHashCode()
{
return ID.GetHashCode();
}
public override bool Equals(object obj)
{
return (obj is Node) && (ID.Equals((obj as Node).ID));
}
}
}

4
http/NetworkApi.cs
View File

@ -34,8 +34,8 @@ namespace ln.skyscanner.http
[Callable]
public Node[] GetNeighbors(string pid)
{
Guid persistenceID = Guid.Parse(pid);
return GlobalNetwork.FindNeighbors(GlobalNetwork.GetNode(persistenceID)).ToArray();
Guid nodeID = Guid.Parse(pid);
return GlobalNetwork.FindNeighbors(GlobalNetwork.GetNode(nodeID)).ToArray();
}
[Callable]