WIP
parent
f8d235309e
commit
b1e7d86aea
|
@ -98,19 +98,22 @@ namespace ln.skyscanner.crawl
|
|||
|
||||
DateTime dateTime = DateTime.Now;
|
||||
|
||||
while (servicesToCheck.Count > 0)
|
||||
RunServiceCheck(servicesToCheck[0]);
|
||||
try
|
||||
{
|
||||
|
||||
//if (CrawledHost.FirstSeen.Equals(DateTime.MinValue))
|
||||
// CrawledHost.FirstSeen = DateTime.Now;
|
||||
while (servicesToCheck.Count > 0)
|
||||
RunServiceCheck(servicesToCheck[0]);
|
||||
|
||||
////CrawledHost.LastSeen = DateTime.Now;
|
||||
Host.LastCheckTime = DateTime.Now - dateTime;
|
||||
Host.LastCheck = dateTime;
|
||||
Host.NextCheck = dateTime + TimeSpan.FromHours(1);
|
||||
} catch (Exception e)
|
||||
{
|
||||
Logging.Log(LogLevel.ERROR, "Crawl.RunJob(): Caught exception: {0}",e);
|
||||
Logging.Log(e);
|
||||
}
|
||||
|
||||
Host.LastCheckTime = DateTime.Now - dateTime;
|
||||
Host.LastCheck = dateTime;
|
||||
Host.NextCheck = dateTime + TimeSpan.FromHours(1);
|
||||
|
||||
bool updated = Crawler.CrawledHosts.Update( Host );
|
||||
bool updated = Crawler.CrawledHosts.Upsert( Host );
|
||||
|
||||
setState("Updating global network");
|
||||
SkyScanner.Instance.Entities.GlobalNetwork.EnqueueUpdate(Host);
|
||||
|
|
|
@ -65,14 +65,22 @@ namespace ln.skyscanner.entities
|
|||
public IEnumerable<Node> FindHostsInSubnet(CIDR network)
|
||||
{
|
||||
IntfIP[] intfIPs = SkyEntities.intfIPCollection.Select((ip) => ip.IP.In(network)).ToArray();
|
||||
NetworkInterface[] networkInterfaces = intfIPs.Select((iip) => iip.NetworkInterface).Distinct().ToArray();
|
||||
Node[] nodes = SkyEntities.nodeCollection.Select(networkInterfaces.Select((ni) => (ODBValue)ni.NodeID).Distinct()).ToArray();
|
||||
NetworkInterface[] networkInterfaces = intfIPs.Select((iip) => iip.NetworkInterface).Where((i)=>i!=null).Distinct().ToArray();
|
||||
Node[] nodes = SkyEntities.nodeCollection.Select(networkInterfaces.Select((ni) => new ODBGuid(ni.NodeID))).ToArray();
|
||||
return nodes;
|
||||
}
|
||||
|
||||
public IEnumerable<Node> FindNeighbors(Node node)
|
||||
{
|
||||
return node.IPAdresses.SelectMany((ip) => Timing.Meassure("FindHostsInSubnet", () => FindHostsInSubnet(ip)).Distinct().ToArray());
|
||||
HashSet<Node> nodes = new HashSet<Node>();
|
||||
foreach (Node neighbor in node.IPAdresses.SelectMany(
|
||||
(ip) => Timing.Meassure("FindHostsInSubnet", () =>
|
||||
FindHostsInSubnet(ip)
|
||||
)
|
||||
))
|
||||
nodes.Add(neighbor);
|
||||
|
||||
return nodes;
|
||||
}
|
||||
|
||||
public Node FindNodeByIP(IEnumerable<CIDR> ips)
|
||||
|
@ -114,16 +122,23 @@ namespace ln.skyscanner.entities
|
|||
}
|
||||
while (true)
|
||||
{
|
||||
CrawledHost crawledHost = null;
|
||||
lock (updateQueue)
|
||||
try
|
||||
{
|
||||
if (updateQueue.Count > 0)
|
||||
crawledHost = updateQueue.Dequeue();
|
||||
else
|
||||
break;
|
||||
}
|
||||
CrawledHost crawledHost = null;
|
||||
lock (updateQueue)
|
||||
{
|
||||
if (updateQueue.Count > 0)
|
||||
crawledHost = updateQueue.Dequeue();
|
||||
else
|
||||
break;
|
||||
}
|
||||
|
||||
Update(crawledHost);
|
||||
Update(crawledHost);
|
||||
} catch (Exception e)
|
||||
{
|
||||
Logging.Log(LogLevel.ERROR, "GlobalNetwork.Update(): Caught Exception: {0}",e);
|
||||
Logging.Log(e);
|
||||
}
|
||||
}
|
||||
|
||||
//SkyScanner.Entities.DBGlobalNetwork.SavePersistent(SkyScanner.Entities.DBGlobalNetwork.Root);
|
||||
|
@ -142,8 +157,17 @@ namespace ln.skyscanner.entities
|
|||
node = new Node(crawledHost.PrimaryIP);
|
||||
node.Name = crawledHost.Name;
|
||||
node.PrimaryMac = crawledHost.PrimaryHWAddr;
|
||||
|
||||
SkyEntities.nodeCollection.Insert(node);
|
||||
}
|
||||
|
||||
if (!crawledHost.PrimaryIP.Equals(node.PrimaryIP))
|
||||
{
|
||||
Logging.Log(LogLevel.INFO,"GlobalNetwork.Update(): will not update Node with unmatched PrimaryIP {0} != {1}",node.PrimaryIP, crawledHost.PrimaryIP);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
foreach (String si in crawledHost.GetHint<string[]>("rfc1213.interfaces",new string[0]))
|
||||
{
|
||||
String[] fields = si.Split(';');
|
||||
|
@ -178,7 +202,6 @@ namespace ln.skyscanner.entities
|
|||
|
||||
if (crawledIPs.Contains(ip))
|
||||
{
|
||||
EnsureSubnet(ip.Network);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
@ -203,6 +226,11 @@ namespace ln.skyscanner.entities
|
|||
}
|
||||
|
||||
|
||||
foreach (CIDR ip in node.IPAdresses)
|
||||
{
|
||||
EnsureSubnet(ip.Network);
|
||||
}
|
||||
|
||||
node.LastUpdate = DateTime.Now;
|
||||
SkyEntities.nodeCollection.Upsert(node);
|
||||
}
|
||||
|
|
|
@ -92,5 +92,15 @@ namespace ln.skyscanner.entities
|
|||
return GetInterface(intfName) != null;
|
||||
}
|
||||
|
||||
public override int GetHashCode()
|
||||
{
|
||||
return ID.GetHashCode();
|
||||
}
|
||||
|
||||
public override bool Equals(object obj)
|
||||
{
|
||||
return (obj is Node) && (ID.Equals((obj as Node).ID));
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
@ -34,8 +34,8 @@ namespace ln.skyscanner.http
|
|||
[Callable]
|
||||
public Node[] GetNeighbors(string pid)
|
||||
{
|
||||
Guid persistenceID = Guid.Parse(pid);
|
||||
return GlobalNetwork.FindNeighbors(GlobalNetwork.GetNode(persistenceID)).ToArray();
|
||||
Guid nodeID = Guid.Parse(pid);
|
||||
return GlobalNetwork.FindNeighbors(GlobalNetwork.GetNode(nodeID)).ToArray();
|
||||
}
|
||||
|
||||
[Callable]
|
||||
|
|
Loading…
Reference in New Issue