ln.skyscanner/entities/GlobalNetwork.cs

using System;
using System.Collections.Generic;
using System.Collections;
using System.Security;
using ln.types;
using System.Linq;
using System.Threading;
using ln.skyscanner.crawl;
using ln.snmp.rfc1213;
using ln.types.odb;
using ln.types.sync;
using ln.logging;
using System.Globalization;
using ln.types.threads;
using ln.types.odb.values;
using ln.types.net;
using System.Runtime.Serialization.Formatters;

namespace ln.skyscanner.entities
{
    public class GlobalNetwork : Persistent
    {
        [Unsynced]
        public SkyScanner SkyScanner => SkyScanner.Instance;
        [Unsynced]
        public SkyEntities SkyEntities { get; private set; }

        public IEnumerable<Subnet> Subnets => SkyEntities.subnetCollection;
        public IEnumerable<Node> Nodes => SkyEntities.nodeCollection;

        public HopMap DefaultHopMap { get; private set; } = new HopMap();

        private object _updateLock = new object();

        public GlobalNetwork()
        {
        }

        public GlobalNetwork(SkyEntities skyEntities)
        {
            SkyEntities = skyEntities;
        }

        public void EnsureSubnet(Network4 network)
        {
            Subnet subnet = SkyEntities.subnetCollection.Where(s => s.Network.Equals(network)).FirstOrDefault();
            if (subnet == null)
            {
                subnet = new Subnet(network);
                SkyEntities.subnetCollection.Upsert(subnet);
            }
        }

        public Node GetNode(Guid id)
        {
            return SkyEntities.nodeCollection.Select(id);
        }

        public Subnet FindSubnetForIP(IPv4 ip)
        {
            return SkyEntities.subnetCollection.Select(net => net.Network.Contains(ip)).FirstOrDefault();
        }

        public IEnumerable<Node> FindHostsInSubnet(Network4 network)
        {
            ODBValue firstIP = ODBMapper.Default.MapValue(network.First);
            ODBValue lastIP = ODBMapper.Default.MapValue(network.Last);

            Query nodeByIpQuery = Query.OR(
                Query.Equals<Node>("Interfaces[].ConfiguredIPs[].Network", network),
                Query.IF<Node>("PrimaryIP", (ip) => ((firstIP <= ip) && (ip <= lastIP)))
                );
            return SkyEntities.nodeCollection.Select(nodeByIpQuery);
        }

        public IEnumerable<Node> FindNeighbors(Node node)
        {
            HashSet<Network4> networks = new HashSet<Network4>();
            HashSet<Node> nodes = new HashSet<Node>();

            if (node.Interfaces.Count > 0)
            {
                foreach (Network4 network in node.Networks)
                    networks.Add(network);
            }
            Subnet subnet = FindSubnetForIP(node.PrimaryIP);
            networks.Add(subnet.Network);

            foreach (Network4 network in networks)
                foreach (Node neighbor in FindHostsInSubnet(network))
                    nodes.Add(neighbor);

            return nodes;
        }

        public Node FindNodeByIP(IEnumerable<IPv4> ips)
        {
            foreach (IPv4 ip in ips)
            {
                Node node = FindNodeByIP(ip);
                if (node != null)
                    return node;
            }
            return null;
        }

        public Node FindNodeByIP(IPv4 ip)
        {
            Node node = SkyEntities.nodeCollection.SelectOne("PrimaryIP", ip);
            if (node == null)
            {
                Query nodeByIpQuery = Query.Equals<Node>("Interfaces[].ConfiguredIPs[].IP", ip);
                node = SkyEntities.nodeCollection.Select(nodeByIpQuery).FirstOrDefault();
            }
            return node;
        }


        [Unsynced]
        private Queue<CrawledHost> updateQueue = new Queue<CrawledHost>();

        public void EnqueueUpdate(CrawledHost crawledHost)
        {
            lock (updateQueue)
            {
                if (!updateQueue.Contains(crawledHost))
                    updateQueue.Enqueue(crawledHost);
            }
        }

        public void Update()
        {
            lock (updateQueue)
            {
                if (updateQueue.Count == 0)
                    return;
            }
            while (true)
            {
                try
                {
                    CrawledHost crawledHost = null;
                    lock (updateQueue)
                    {
                        if (updateQueue.Count > 0)
                            crawledHost = updateQueue.Dequeue();
                        else
                            break;
                    }

                    Update(crawledHost);
                } catch (Exception e)
                {
                    Logging.Log(LogLevel.ERROR, "GlobalNetwork.Update(): Caught Exception: {0}",e);
                    Logging.Log(e);
                }
            }
        }

        private void Update(CrawledHost crawledHost)
        {
            Logging.Log(LogLevel.DEBUG, "Entity: update {0}", crawledHost.PrimaryIP);

            Node node = FindNodeByIP(crawledHost.PrimaryIP);
            if (node == null)
                node = FindNodeByIP(crawledHost.IPAddresses);

            if (node == null)
            {
                node = new Node(crawledHost.PrimaryIP);
                node.Name = crawledHost.Name;
                node.PrimaryMac = crawledHost.PrimaryHWAddr;

                SkyEntities.nodeCollection.Insert(node);
            }

            if (!crawledHost.PrimaryIP.Equals(node.PrimaryIP))
            {
                Logging.Log(LogLevel.INFO,"GlobalNetwork.Update(): will not update Node with unmatched PrimaryIP {0} != {1}",node.PrimaryIP, crawledHost.PrimaryIP);
                return;
            }


            foreach (String si in crawledHost.GetHint<string[]>("rfc1213.interfaces",new string[0]))
            {
                String[] fields = si.Split(';');

                NetworkInterface networkInterface = node.GetInterface(fields[0]);
                if (networkInterface == null)
                {
                    networkInterface = new NetworkInterface(fields[0]);
                    node.Interfaces.Add(networkInterface);
                }

                networkInterface.HWAddress = fields[1];

                IPv4[][] crawledIPs = fields[2].Split(',').Where((sip) => !String.Empty.Equals(sip))
                    .Select((sip) => sip.Split('/').Select(ip => IPv4.Parse(ip)).ToArray()).ToArray();

                ConfiguredIP[] currentIPs  = networkInterface.ConfiguredIPs.ToArray();

                foreach (IPv4[] crawledIP in crawledIPs)
                {
                    int n;
                    for (n=0; n < currentIPs.Length; n++)
                        if (currentIPs[n].IP.Equals(crawledIP[0]))
                            break;
                    if (n == currentIPs.Length)
                    {
                        ConfiguredIP miss = new ConfiguredIP(crawledIP[0], new Network4(crawledIP[0], crawledIP[1]));
                        networkInterface.ConfiguredIPs.Add(miss);
                    }
                }

                foreach (ConfiguredIP iip in currentIPs)
                {
                    IPv4 ip = iip.IP;

                    if (!crawledIPs.Select((ipl)=>ipl[0]).Contains(ip))
                    {
                        networkInterface.ConfiguredIPs.Remove(iip);
                    }
                }
            }

            node.RemoveURI("ssh");
            if (crawledHost.GetHint<int>("ssh.port", -1) != -1)
            {
                node.AddURI(
                    new URI(
                        String.Format("ssh://{0}:{1}@{2}:{3}",
                            crawledHost.GetHint<string>("ssh.login"),
                            crawledHost.GetHint<string>("ssh.password"),
                            crawledHost.GetHint<IPv4>("ssh.ip"),
                            crawledHost.GetHint<int>("ssh.port")
                            )
                        )
                    );
            }


            node.RemoveURI("http");
            if (crawledHost.GetHint("http.port",-1) != -1)
            {
                node.AddURI(new URI(String.Format("http://{0}:{1}",
                        crawledHost.GetHint<IPv4>("http.ip", null),
                        crawledHost.GetHint<int>("http.port"))
                        ));
            }


            string httpIndex = crawledHost.GetHint("http.content", "");

            if (httpIndex.Contains("<title>RouterOS router configuration page</title>"))
            {
                node.Vendor = "MicroTik";
            }
            if (crawledHost.GetHint("http.server", "").Equals("Viprinet"))
                node.Vendor = "Viprinet";

            if (crawledHost.GetHint("snmp.orids",new string[0]).Contains("1.3.6.1.4.1.41112"))
            {
                node.Vendor = "Ubiquity";
            }

            foreach (Network4 network in node.Networks)
                EnsureSubnet(network);

            node.LastUpdate = DateTime.Now;
            SkyEntities.nodeCollection.Upsert(node);
        }
    }
}