diff --git a/crawl/Crawl.cs b/crawl/Crawl.cs index 231b177..78c6f2d 100644 --- a/crawl/Crawl.cs +++ b/crawl/Crawl.cs @@ -98,19 +98,22 @@ namespace ln.skyscanner.crawl DateTime dateTime = DateTime.Now; - while (servicesToCheck.Count > 0) - RunServiceCheck(servicesToCheck[0]); + try + { - //if (CrawledHost.FirstSeen.Equals(DateTime.MinValue)) - // CrawledHost.FirstSeen = DateTime.Now; + while (servicesToCheck.Count > 0) + RunServiceCheck(servicesToCheck[0]); - ////CrawledHost.LastSeen = DateTime.Now; + Host.LastCheckTime = DateTime.Now - dateTime; + Host.LastCheck = dateTime; + Host.NextCheck = dateTime + TimeSpan.FromHours(1); + } catch (Exception e) + { + Logging.Log(LogLevel.ERROR, "Crawl.RunJob(): Caught exception: {0}",e); + Logging.Log(e); + } - Host.LastCheckTime = DateTime.Now - dateTime; - Host.LastCheck = dateTime; - Host.NextCheck = dateTime + TimeSpan.FromHours(1); - - bool updated = Crawler.CrawledHosts.Update( Host ); + bool updated = Crawler.CrawledHosts.Upsert( Host ); setState("Updating global network"); SkyScanner.Instance.Entities.GlobalNetwork.EnqueueUpdate(Host); diff --git a/entities/GlobalNetwork.cs b/entities/GlobalNetwork.cs index e9ca68f..b413128 100644 --- a/entities/GlobalNetwork.cs +++ b/entities/GlobalNetwork.cs @@ -65,14 +65,22 @@ namespace ln.skyscanner.entities public IEnumerable FindHostsInSubnet(CIDR network) { IntfIP[] intfIPs = SkyEntities.intfIPCollection.Select((ip) => ip.IP.In(network)).ToArray(); - NetworkInterface[] networkInterfaces = intfIPs.Select((iip) => iip.NetworkInterface).Distinct().ToArray(); - Node[] nodes = SkyEntities.nodeCollection.Select(networkInterfaces.Select((ni) => (ODBValue)ni.NodeID).Distinct()).ToArray(); + NetworkInterface[] networkInterfaces = intfIPs.Select((iip) => iip.NetworkInterface).Where((i)=>i!=null).Distinct().ToArray(); + Node[] nodes = SkyEntities.nodeCollection.Select(networkInterfaces.Select((ni) => new ODBGuid(ni.NodeID))).ToArray(); return nodes; } public IEnumerable FindNeighbors(Node node) { - return node.IPAdresses.SelectMany((ip) => Timing.Meassure("FindHostsInSubnet", () => FindHostsInSubnet(ip)).Distinct().ToArray()); + HashSet nodes = new HashSet(); + foreach (Node neighbor in node.IPAdresses.SelectMany( + (ip) => Timing.Meassure("FindHostsInSubnet", () => + FindHostsInSubnet(ip) + ) + )) + nodes.Add(neighbor); + + return nodes; } public Node FindNodeByIP(IEnumerable ips) @@ -114,16 +122,23 @@ namespace ln.skyscanner.entities } while (true) { - CrawledHost crawledHost = null; - lock (updateQueue) + try { - if (updateQueue.Count > 0) - crawledHost = updateQueue.Dequeue(); - else - break; - } + CrawledHost crawledHost = null; + lock (updateQueue) + { + if (updateQueue.Count > 0) + crawledHost = updateQueue.Dequeue(); + else + break; + } - Update(crawledHost); + Update(crawledHost); + } catch (Exception e) + { + Logging.Log(LogLevel.ERROR, "GlobalNetwork.Update(): Caught Exception: {0}",e); + Logging.Log(e); + } } //SkyScanner.Entities.DBGlobalNetwork.SavePersistent(SkyScanner.Entities.DBGlobalNetwork.Root); @@ -142,8 +157,17 @@ namespace ln.skyscanner.entities node = new Node(crawledHost.PrimaryIP); node.Name = crawledHost.Name; node.PrimaryMac = crawledHost.PrimaryHWAddr; + + SkyEntities.nodeCollection.Insert(node); } + if (!crawledHost.PrimaryIP.Equals(node.PrimaryIP)) + { + Logging.Log(LogLevel.INFO,"GlobalNetwork.Update(): will not update Node with unmatched PrimaryIP {0} != {1}",node.PrimaryIP, crawledHost.PrimaryIP); + return; + } + + foreach (String si in crawledHost.GetHint("rfc1213.interfaces",new string[0])) { String[] fields = si.Split(';'); @@ -178,7 +202,6 @@ namespace ln.skyscanner.entities if (crawledIPs.Contains(ip)) { - EnsureSubnet(ip.Network); } else { @@ -203,6 +226,11 @@ namespace ln.skyscanner.entities } + foreach (CIDR ip in node.IPAdresses) + { + EnsureSubnet(ip.Network); + } + node.LastUpdate = DateTime.Now; SkyEntities.nodeCollection.Upsert(node); } diff --git a/entities/Node.cs b/entities/Node.cs index 5714cc5..25ba073 100644 --- a/entities/Node.cs +++ b/entities/Node.cs @@ -92,5 +92,15 @@ namespace ln.skyscanner.entities return GetInterface(intfName) != null; } + public override int GetHashCode() + { + return ID.GetHashCode(); + } + + public override bool Equals(object obj) + { + return (obj is Node) && (ID.Equals((obj as Node).ID)); + } + } } diff --git a/http/NetworkApi.cs b/http/NetworkApi.cs index 03d9bc4..af13c91 100644 --- a/http/NetworkApi.cs +++ b/http/NetworkApi.cs @@ -34,8 +34,8 @@ namespace ln.skyscanner.http [Callable] public Node[] GetNeighbors(string pid) { - Guid persistenceID = Guid.Parse(pid); - return GlobalNetwork.FindNeighbors(GlobalNetwork.GetNode(persistenceID)).ToArray(); + Guid nodeID = Guid.Parse(pid); + return GlobalNetwork.FindNeighbors(GlobalNetwork.GetNode(nodeID)).ToArray(); } [Callable]