using System; using System.Collections.Generic; using System.Collections; using System.Security; using ln.types; using System.Linq; using System.Threading; using ln.skyscanner.crawl; using ln.snmp.rfc1213; using ln.types.odb; using ln.types.sync; using ln.logging; using System.Globalization; using ln.types.threads; using ln.types.odb.values; using ln.types.net; namespace ln.skyscanner.entities { public class GlobalNetwork : Persistent { [Unsynced] public SkyScanner SkyScanner => SkyScanner.Instance; [Unsynced] public SkyEntities SkyEntities { get; private set; } public IEnumerable Subnets => SkyEntities.subnetCollection; public IEnumerable Nodes => SkyEntities.nodeCollection; public HopMap DefaultHopMap { get; private set; } = new HopMap(); private object _updateLock = new object(); public GlobalNetwork() { } public GlobalNetwork(SkyEntities skyEntities) { SkyEntities = skyEntities; } public void EnsureSubnet(Network4 network) { SkyScanner.Crawler.EnsureSubnet(network); Subnet subnet = SkyEntities.subnetCollection.Where(s => s.Network.Equals(network)).FirstOrDefault(); if (subnet == null) { subnet = new Subnet(network); SkyEntities.subnetCollection.Upsert(subnet); } } public Node GetNode(Guid id) { return SkyEntities.nodeCollection.Select(id); } public Subnet FindSubnetForIP(IPv4 ip) { return SkyEntities.subnetCollection.Select(net => net.Network.Contains(ip)).FirstOrDefault(); } public IEnumerable FindHostsInSubnet(Network4 network) { Query nodeByIpQuery = Query.Equals("Interfaces[].ConfiguredIPs[].Network", network); return SkyEntities.nodeCollection.Select(nodeByIpQuery); } public IEnumerable FindNeighbors(Node node) { HashSet nodes = new HashSet(); if (node.Interfaces.Count > 0) { foreach (Network4 network in node.Networks) { foreach (Node neighbor in FindHostsInSubnet(network)) nodes.Add(neighbor); } } else { Subnet subnet = FindSubnetForIP(node.PrimaryIP); IEnumerable nodelist = FindHostsInSubnet(subnet.Network); foreach (Node n in nodelist) nodes.Add(n); } return nodes; } public Node FindNodeByIP(IEnumerable ips) { foreach (IPv4 ip in ips) { Node node = FindNodeByIP(ip); if (node != null) return node; } return null; } public Node FindNodeByIP(IPv4 ip) { Node node = SkyEntities.nodeCollection.SelectOne("PrimaryIP", ip); if (node == null) { Query nodeByIpQuery = Query.Equals("Interfaces[].ConfiguredIPs[].IP", ip); node = SkyEntities.nodeCollection.Select(nodeByIpQuery).FirstOrDefault(); } return node; } [Unsynced] private Queue updateQueue = new Queue(); public void EnqueueUpdate(CrawledHost crawledHost) { lock (updateQueue) { if (!updateQueue.Contains(crawledHost)) updateQueue.Enqueue(crawledHost); } } public void Update() { lock (updateQueue) { if (updateQueue.Count == 0) return; } while (true) { try { CrawledHost crawledHost = null; lock (updateQueue) { if (updateQueue.Count > 0) crawledHost = updateQueue.Dequeue(); else break; } Update(crawledHost); } catch (Exception e) { Logging.Log(LogLevel.ERROR, "GlobalNetwork.Update(): Caught Exception: {0}",e); Logging.Log(e); } } } private void Update(CrawledHost crawledHost) { Logging.Log(LogLevel.DEBUG, "Entity: update {0}", crawledHost.PrimaryIP); Node node = FindNodeByIP(crawledHost.PrimaryIP); if (node == null) node = FindNodeByIP(crawledHost.IPAddresses); if (node == null) { node = new Node(crawledHost.PrimaryIP); node.Name = crawledHost.Name; node.PrimaryMac = crawledHost.PrimaryHWAddr; SkyEntities.nodeCollection.Insert(node); } if (!crawledHost.PrimaryIP.Equals(node.PrimaryIP)) { Logging.Log(LogLevel.INFO,"GlobalNetwork.Update(): will not update Node with unmatched PrimaryIP {0} != {1}",node.PrimaryIP, crawledHost.PrimaryIP); return; } foreach (String si in crawledHost.GetHint("rfc1213.interfaces",new string[0])) { String[] fields = si.Split(';'); NetworkInterface networkInterface = node.GetInterface(fields[0]); if (networkInterface == null) { networkInterface = new NetworkInterface(fields[0]); node.Interfaces.Add(networkInterface); } networkInterface.HWAddress = fields[1]; IPv4[][] crawledIPs = fields[2].Split(',').Where((sip) => !String.Empty.Equals(sip)) .Select((sip) => sip.Split('/').Select(ip => IPv4.Parse(ip)).ToArray()).ToArray(); ConfiguredIP[] currentIPs = networkInterface.ConfiguredIPs.ToArray(); foreach (IPv4[] crawledIP in crawledIPs) { int n; for (n=0; n < currentIPs.Length; n++) if (currentIPs[n].IP.Equals(crawledIP[0])) break; if (n == currentIPs.Length) { ConfiguredIP miss = new ConfiguredIP(crawledIP[0], new Network4(crawledIP[0], crawledIP[1])); networkInterface.ConfiguredIPs.Add(miss); } } foreach (ConfiguredIP iip in currentIPs) { IPv4 ip = iip.IP; if (!crawledIPs.Select((ipl)=>ipl[0]).Contains(ip)) { networkInterface.ConfiguredIPs.Remove(iip); } } } node.RemoveURI("ssh"); if (crawledHost.GetHint("ssh.port", -1) != -1) { node.AddURI( new URI( String.Format("ssh://{0}:{1}@{2}:{3}", crawledHost.GetHint("ssh.login"), crawledHost.GetHint("ssh.password"), crawledHost.GetHint("ssh.ip"), crawledHost.GetHint("ssh.port") ) ) ); } node.RemoveURI("http"); if (crawledHost.GetHint("http.port",-1) != -1) { node.AddURI(new URI(String.Format("http://{0}:{1}", crawledHost.GetHint("http.ip", null), crawledHost.GetHint("http.port")) )); } string httpIndex = crawledHost.GetHint("http.content", ""); if (httpIndex.Contains("RouterOS router configuration page")) { node.Vendor = "MicroTik"; } if (crawledHost.GetHint("snmp.orids",new string[0]).Contains("1.3.6.1.4.1.41112")) { node.Vendor = "Ubiquity"; } foreach (Network4 network in node.Networks) EnsureSubnet(network); node.LastUpdate = DateTime.Now; SkyEntities.nodeCollection.Upsert(node); } } }