WIP
parent
f8d235309e
commit
b1e7d86aea
|
@ -98,19 +98,22 @@ namespace ln.skyscanner.crawl
|
||||||
|
|
||||||
DateTime dateTime = DateTime.Now;
|
DateTime dateTime = DateTime.Now;
|
||||||
|
|
||||||
while (servicesToCheck.Count > 0)
|
try
|
||||||
RunServiceCheck(servicesToCheck[0]);
|
{
|
||||||
|
|
||||||
//if (CrawledHost.FirstSeen.Equals(DateTime.MinValue))
|
while (servicesToCheck.Count > 0)
|
||||||
// CrawledHost.FirstSeen = DateTime.Now;
|
RunServiceCheck(servicesToCheck[0]);
|
||||||
|
|
||||||
////CrawledHost.LastSeen = DateTime.Now;
|
Host.LastCheckTime = DateTime.Now - dateTime;
|
||||||
|
Host.LastCheck = dateTime;
|
||||||
|
Host.NextCheck = dateTime + TimeSpan.FromHours(1);
|
||||||
|
} catch (Exception e)
|
||||||
|
{
|
||||||
|
Logging.Log(LogLevel.ERROR, "Crawl.RunJob(): Caught exception: {0}",e);
|
||||||
|
Logging.Log(e);
|
||||||
|
}
|
||||||
|
|
||||||
Host.LastCheckTime = DateTime.Now - dateTime;
|
bool updated = Crawler.CrawledHosts.Upsert( Host );
|
||||||
Host.LastCheck = dateTime;
|
|
||||||
Host.NextCheck = dateTime + TimeSpan.FromHours(1);
|
|
||||||
|
|
||||||
bool updated = Crawler.CrawledHosts.Update( Host );
|
|
||||||
|
|
||||||
setState("Updating global network");
|
setState("Updating global network");
|
||||||
SkyScanner.Instance.Entities.GlobalNetwork.EnqueueUpdate(Host);
|
SkyScanner.Instance.Entities.GlobalNetwork.EnqueueUpdate(Host);
|
||||||
|
|
|
@ -65,14 +65,22 @@ namespace ln.skyscanner.entities
|
||||||
public IEnumerable<Node> FindHostsInSubnet(CIDR network)
|
public IEnumerable<Node> FindHostsInSubnet(CIDR network)
|
||||||
{
|
{
|
||||||
IntfIP[] intfIPs = SkyEntities.intfIPCollection.Select((ip) => ip.IP.In(network)).ToArray();
|
IntfIP[] intfIPs = SkyEntities.intfIPCollection.Select((ip) => ip.IP.In(network)).ToArray();
|
||||||
NetworkInterface[] networkInterfaces = intfIPs.Select((iip) => iip.NetworkInterface).Distinct().ToArray();
|
NetworkInterface[] networkInterfaces = intfIPs.Select((iip) => iip.NetworkInterface).Where((i)=>i!=null).Distinct().ToArray();
|
||||||
Node[] nodes = SkyEntities.nodeCollection.Select(networkInterfaces.Select((ni) => (ODBValue)ni.NodeID).Distinct()).ToArray();
|
Node[] nodes = SkyEntities.nodeCollection.Select(networkInterfaces.Select((ni) => new ODBGuid(ni.NodeID))).ToArray();
|
||||||
return nodes;
|
return nodes;
|
||||||
}
|
}
|
||||||
|
|
||||||
public IEnumerable<Node> FindNeighbors(Node node)
|
public IEnumerable<Node> FindNeighbors(Node node)
|
||||||
{
|
{
|
||||||
return node.IPAdresses.SelectMany((ip) => Timing.Meassure("FindHostsInSubnet", () => FindHostsInSubnet(ip)).Distinct().ToArray());
|
HashSet<Node> nodes = new HashSet<Node>();
|
||||||
|
foreach (Node neighbor in node.IPAdresses.SelectMany(
|
||||||
|
(ip) => Timing.Meassure("FindHostsInSubnet", () =>
|
||||||
|
FindHostsInSubnet(ip)
|
||||||
|
)
|
||||||
|
))
|
||||||
|
nodes.Add(neighbor);
|
||||||
|
|
||||||
|
return nodes;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Node FindNodeByIP(IEnumerable<CIDR> ips)
|
public Node FindNodeByIP(IEnumerable<CIDR> ips)
|
||||||
|
@ -114,16 +122,23 @@ namespace ln.skyscanner.entities
|
||||||
}
|
}
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
CrawledHost crawledHost = null;
|
try
|
||||||
lock (updateQueue)
|
|
||||||
{
|
{
|
||||||
if (updateQueue.Count > 0)
|
CrawledHost crawledHost = null;
|
||||||
crawledHost = updateQueue.Dequeue();
|
lock (updateQueue)
|
||||||
else
|
{
|
||||||
break;
|
if (updateQueue.Count > 0)
|
||||||
}
|
crawledHost = updateQueue.Dequeue();
|
||||||
|
else
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
Update(crawledHost);
|
Update(crawledHost);
|
||||||
|
} catch (Exception e)
|
||||||
|
{
|
||||||
|
Logging.Log(LogLevel.ERROR, "GlobalNetwork.Update(): Caught Exception: {0}",e);
|
||||||
|
Logging.Log(e);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//SkyScanner.Entities.DBGlobalNetwork.SavePersistent(SkyScanner.Entities.DBGlobalNetwork.Root);
|
//SkyScanner.Entities.DBGlobalNetwork.SavePersistent(SkyScanner.Entities.DBGlobalNetwork.Root);
|
||||||
|
@ -142,8 +157,17 @@ namespace ln.skyscanner.entities
|
||||||
node = new Node(crawledHost.PrimaryIP);
|
node = new Node(crawledHost.PrimaryIP);
|
||||||
node.Name = crawledHost.Name;
|
node.Name = crawledHost.Name;
|
||||||
node.PrimaryMac = crawledHost.PrimaryHWAddr;
|
node.PrimaryMac = crawledHost.PrimaryHWAddr;
|
||||||
|
|
||||||
|
SkyEntities.nodeCollection.Insert(node);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!crawledHost.PrimaryIP.Equals(node.PrimaryIP))
|
||||||
|
{
|
||||||
|
Logging.Log(LogLevel.INFO,"GlobalNetwork.Update(): will not update Node with unmatched PrimaryIP {0} != {1}",node.PrimaryIP, crawledHost.PrimaryIP);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
foreach (String si in crawledHost.GetHint<string[]>("rfc1213.interfaces",new string[0]))
|
foreach (String si in crawledHost.GetHint<string[]>("rfc1213.interfaces",new string[0]))
|
||||||
{
|
{
|
||||||
String[] fields = si.Split(';');
|
String[] fields = si.Split(';');
|
||||||
|
@ -178,7 +202,6 @@ namespace ln.skyscanner.entities
|
||||||
|
|
||||||
if (crawledIPs.Contains(ip))
|
if (crawledIPs.Contains(ip))
|
||||||
{
|
{
|
||||||
EnsureSubnet(ip.Network);
|
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
@ -203,6 +226,11 @@ namespace ln.skyscanner.entities
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
foreach (CIDR ip in node.IPAdresses)
|
||||||
|
{
|
||||||
|
EnsureSubnet(ip.Network);
|
||||||
|
}
|
||||||
|
|
||||||
node.LastUpdate = DateTime.Now;
|
node.LastUpdate = DateTime.Now;
|
||||||
SkyEntities.nodeCollection.Upsert(node);
|
SkyEntities.nodeCollection.Upsert(node);
|
||||||
}
|
}
|
||||||
|
|
|
@ -92,5 +92,15 @@ namespace ln.skyscanner.entities
|
||||||
return GetInterface(intfName) != null;
|
return GetInterface(intfName) != null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public override int GetHashCode()
|
||||||
|
{
|
||||||
|
return ID.GetHashCode();
|
||||||
|
}
|
||||||
|
|
||||||
|
public override bool Equals(object obj)
|
||||||
|
{
|
||||||
|
return (obj is Node) && (ID.Equals((obj as Node).ID));
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -34,8 +34,8 @@ namespace ln.skyscanner.http
|
||||||
[Callable]
|
[Callable]
|
||||||
public Node[] GetNeighbors(string pid)
|
public Node[] GetNeighbors(string pid)
|
||||||
{
|
{
|
||||||
Guid persistenceID = Guid.Parse(pid);
|
Guid nodeID = Guid.Parse(pid);
|
||||||
return GlobalNetwork.FindNeighbors(GlobalNetwork.GetNode(persistenceID)).ToArray();
|
return GlobalNetwork.FindNeighbors(GlobalNetwork.GetNode(nodeID)).ToArray();
|
||||||
}
|
}
|
||||||
|
|
||||||
[Callable]
|
[Callable]
|
||||||
|
|
Loading…
Reference in New Issue