Skytron
/
ln.skyscanner
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Broken WIP

broken
Harald Wolff 2019-08-29 13:14:52 +02:00
parent 8c7e2bc5d3
commit 895231b820
44 changed files with 2955 additions and 2034 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
Program.cs
View File

@ -7,44 +7,16 @@
// * // *
// * // *
// **/ // **/
using System;
using System.Collections.Generic;
using ln.snmp;
using ln.logging; using ln.logging;
using ln.skyscanner.entities;
using ln.types.threads;
using ln.types.net;
using ln.types.odb;
using ln.types.odb.index;
using ln.types.odb.values;
using ln.skyscanner.checks;
using System.Linq;
using ln.skyscanner.import.skytron;
namespace ln.skyscanner namespace ln.skyscanner
{ {
class MainClass class MainClass
{ {
public static void Main(string[] args) public static void Main(string[] args)
{ {
FileLogger fileLogger = new FileLogger("skyscanner.log");
fileLogger.MaxLogLevel = LogLevel.INFO;
Logger.Default.Backends.Add(fileLogger);
Logger.ConsoleLogger.MaxLogLevel = LogLevel.INFO; Logger.ConsoleLogger.MaxLogLevel = LogLevel.INFO;
new SkyScanner().Start(args);
SkyScanner skyScanner = new SkyScanner();
Initialize();
skyScanner.Start(args);
}
private static void Initialize()
{
// SNMPEngine.DefaultEngine.Timeout = 3500;
} }
} }

131
SkyEntities.cs
View File

@ -1,131 +0,0 @@
// /**
// * File: SkyEntities.cs
// * Author: haraldwolff
// *
// * This file and it's content is copyrighted by the Author and / or copyright holder.
// * Any use wihtout proper permission is illegal and may lead to legal actions.
// *
// *
// **/
using System;
using ln.types.odb;
using ln.skyscanner.entities;
using System.IO;
using ln.http.resources;
using System.Linq;
using ln.types.odb.mapped;
using ln.types.net;
using ln.skyscanner.crawl;
using ln.skyscanner.checks;
using ln.logging;
using System.Collections.Generic;
namespace ln.skyscanner
{
public class SkyEntities
{
public SkyScanner SkyScanner { get; }
public string BasePath => Path.Combine(SkyScanner.BasePath, "entities");
public GlobalNetwork GlobalNetwork { get; private set; }
public ODB ODB { get; private set; }
public ODBCollection<Node> NodeCollection { get; private set; }
public ODBCollection<Subnet> SubnetCollection { get; private set; }
public ODBCollection<PointOfPresence> PointOfPresenceCollection { get; private set; }
public ODBCollection<L2Segment> L2SegmentCollection { get; private set; }
public ODBCollection<CrawledHost> CrawledHosts { get; private set; }
public ODBCollection<CrawledSubnet> CrawledSubnets { get; private set; }
public ODBCollection<Network4> BlockedNetworks { get; private set; }
public ODBCollection<SkyCheckState> SkyCheckStates { get; private set; }
public SkyEntities(SkyScanner skyScanner)
{
SkyScanner = skyScanner;
Logging.Log(LogLevel.INFO, "SkyEntities: initializing");
ODB = new ODB(BasePath);
NodeCollection = ODB.GetCollection<Node>();
NodeCollection.EnableStrongCache(true);
NodeCollection.EnsureIndex("uniqueIdentity", true);
NodeCollection.EnsureIndeces(
"PrimaryIP",
"Interfaces[].ConfiguredIPs[].IP",
"Interfaces[].ConfiguredIPs[].Network"
);
Network4 n192 = Network4.Parse("192.168.0.0/16");
/* Preload all nodes to increase load speed*/
foreach (Node node in NodeCollection.ToArray())
{
//if (n192.Contains(node.PrimaryIP))
//NodeCollection.Delete(node);
}
SubnetCollection = ODB.GetCollection<Subnet>();
SubnetCollection.EnsureIndeces("Network");
PointOfPresenceCollection = ODB.GetCollection<PointOfPresence>();
PointOfPresenceCollection.EnsureIndeces("ForeignName");
L2SegmentCollection = ODB.GetCollection<L2Segment>();
L2SegmentCollection.EnsureIndeces("Network","PoPs");
CrawledHosts = ODB.GetCollection<CrawledHost>();
CrawledHosts.EnsureIndeces("PrimaryIP","IPAddresses[]");
CrawledSubnets = ODB.GetCollection<CrawledSubnet>();
BlockedNetworks = ODB.GetCollection<Network4>("blockedNetworks");
SkyCheckStates = ODB.GetCollection<SkyCheckState>();
SkyCheckStates.EnableStrongCache(true);
SkyCheckStates.EnsureIndeces("Node");
SkyCheckStates.EnsureUniqueness("Node", "CheckName");
//foreach (SkyCheckState checkState in SkyCheckStates.ToArray())
//{
// if (object.ReferenceEquals(checkState.Node, null))
// SkyCheckStates.Delete(checkState);
//}
Logging.Log(LogLevel.INFO, "SkyEntities: initialized");
GlobalNetwork = new GlobalNetwork();
}
public PerformanceValue LookupPerformanceValue(String perfName)
{
string[] resourcePath = perfName.Split('/');
Node node = NodeCollection.Query("uniqueIdentity", resourcePath[0]).FirstOrDefault();
if (node == null)
throw new KeyNotFoundException();
SkyCheckState skyCheckState = SkyCheckStates.Query(
Query.AND(
Query.Equals<SkyCheckState>("Node", node.ID),
Query.Equals<SkyCheckState>("CheckName", resourcePath[1])
)
).FirstOrDefault();
if (skyCheckState == null)
throw new KeyNotFoundException();
foreach (PerformanceValue performanceValue in skyCheckState.PerformanceValues)
if (performanceValue.PerfName.Equals(perfName))
return performanceValue;
throw new KeyNotFoundException();
}
}
}

18
SkyScanner.cs
View File

@ -4,7 +4,6 @@ using ln.http;
using System.Net; using System.Net;
using ln.skyscanner.http; using ln.skyscanner.http;
using System.IO; using System.IO;
using ln.skyscanner.crawl;
using ln.types.threads; using ln.types.threads;
using ln.skyscanner.checks; using ln.skyscanner.checks;
using ln.skyscanner.import.skytron; using ln.skyscanner.import.skytron;
@ -27,17 +26,8 @@ namespace ln.skyscanner
public static SkyScanner Instance { get; private set; } public static SkyScanner Instance { get; private set; }
public String BasePath { get; private set; } public String BasePath { get; private set; }
public Pool ConveniencePool { get; } public Pool ConveniencePool { get; }
public HTTPServer HTTPServer { get; private set; }
public Crawler Crawler { get; private set; }
public SkyEntities Entities { get; private set; }
public SkyChecker Checker { get; private set; }
public bool OptionDisableChecker { get; private set; }
public String OptionCrawlVendor { get; private set; }
public SkyScanner() public SkyScanner()
{ {
if (Instance != null) if (Instance != null)
@ -59,7 +49,6 @@ namespace ln.skyscanner
BasePath = Arguments["base-path"].Value; BasePath = Arguments["base-path"].Value;
Logging.Log(LogLevel.INFO, "SkyScanner: BasePath={0}", BasePath); Logging.Log(LogLevel.INFO, "SkyScanner: BasePath={0}", BasePath);
ConveniencePool = new Pool(); ConveniencePool = new Pool();
} }
@ -67,6 +56,10 @@ namespace ln.skyscanner
{ {
base.PrepareStart(); base.PrepareStart();
FileLogger fileLogger = new FileLogger("skyscanner.log");
fileLogger.MaxLogLevel = (LogLevel)Enum.Parse(typeof(LogLevel), Arguments["log-level"].Value);
Logger.Default.Backends.Add(fileLogger);
if (Arguments["import-skytron"].IsSet) if (Arguments["import-skytron"].IsSet)
{ {
SkytronImport si = new SkytronImport(Arguments["import-skytron"].Value); SkytronImport si = new SkytronImport(Arguments["import-skytron"].Value);
@ -104,6 +97,7 @@ namespace ln.skyscanner
ServiceContainer.Add(ServiceDefinition.From<EntityService>(true)); ServiceContainer.Add(ServiceDefinition.From<EntityService>(true));
ServiceContainer.Add(ServiceDefinition.From<CheckService>(true)); ServiceContainer.Add(ServiceDefinition.From<CheckService>(true));
ServiceContainer.Add(ServiceDefinition.From<CrawlService>(true)); ServiceContainer.Add(ServiceDefinition.From<CrawlService>(true));
ServiceContainer.Add(ServiceDefinition.From<PerformanceValueService>(true));
} }
@ -254,7 +248,7 @@ namespace ln.skyscanner
else else
{ {
Logging.Log(LogLevel.INFO, "DebugCheck(): Node: {0}",JSONObject.From(node).ToString()); Logging.Log(LogLevel.INFO, "DebugCheck(): Node: {0}",JSONObject.From(node).ToString());
CheckJob checkJob = new CheckJob(node); CheckJob checkJob = new CheckJob(null, node);
Logging.Log(LogLevel.INFO, "Prepare..."); Logging.Log(LogLevel.INFO, "Prepare...");
checkJob.Prepare(); checkJob.Prepare();

27
checks/APC.cs
View File

@ -7,6 +7,7 @@ using ln.snmp.types;
using ln.logging; using ln.logging;
using Microsoft.Win32.SafeHandles; using Microsoft.Win32.SafeHandles;
using Renci.SshNet.Messages.Connection; using Renci.SshNet.Messages.Connection;
using ln.skyscanner.services;
namespace ln.skyscanner.checks namespace ln.skyscanner.checks
{ {
@ -18,11 +19,11 @@ namespace ln.skyscanner.checks
} }
public override void Check(SkyChecker skyChecker,ref SkyCheckState checkState,Node node) public override void Check(CheckService checkService, ref SkyCheckState checkState,Node node)
{ {
foreach (URI snmpUri in node.FindURIs("snmp")) foreach (URI snmpUri in node.FindURIs("snmp"))
{ {
using (SnmpInterface snmp = SnmpInterface.FromURI(snmpUri,skyChecker.SNMPEngine)) using (SnmpInterface snmp = SnmpInterface.FromURI(snmpUri,checkService.SNMPEngine))
{ {
try try
{ {
@ -51,29 +52,29 @@ namespace ln.skyscanner.checks
double voltage = (double)((Integer)(input[1].Items[1])).LongValue; double voltage = (double)((Integer)(input[1].Items[1])).LongValue;
double current = (double)((Integer)(input[2].Items[1])).LongValue; double current = (double)((Integer)(input[2].Items[1])).LongValue;
checkState.WritePerformanceValue(String.Format("ups_input_{0}_frequency", n), frequency, wLower: 48, wUpper: 52, cLower: 45, cUpper: 55); node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_input_{0}_frequency", n), frequency, wLower: 48, wUpper: 52, cLower: 45, cUpper: 55);
checkState.WritePerformanceValue(String.Format("ups_input_{0}_voltage", n), voltage, wLower: 218.5, wUpper: 238, cLower: 212, cUpper: 245); node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_input_{0}_voltage", n), voltage, wLower: 218.5, wUpper: 238, cLower: 212, cUpper: 245);
checkState.WritePerformanceValue(String.Format("ups_input_{0}_current", n), current); node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_input_{0}_current", n), current);
n++; n++;
} }
n = 0; n = 0;
foreach (Sequence[] output in outputs) foreach (Sequence[] output in outputs)
{ {
checkState.WritePerformanceValue(String.Format("ups_output_{0}_voltage", n), (double)((Integer)(output[0].Items[1])).LongValue, wLower: 218.5, wUpper: 238, cLower: 212, cUpper: 245); node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_output_{0}_voltage", n), (double)((Integer)(output[0].Items[1])).LongValue, wLower: 218.5, wUpper: 238, cLower: 212, cUpper: 245);
checkState.WritePerformanceValue(String.Format("ups_output_{0}_current", n), (double)((Integer)(output[1].Items[1])).LongValue / 10.0); node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_output_{0}_current", n), (double)((Integer)(output[1].Items[1])).LongValue / 10.0);
checkState.WritePerformanceValue(String.Format("ups_output_{0}_load", n), (double)((Integer)(output[2].Items[1])).LongValue, wUpper: 50, cUpper: 75); node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_output_{0}_load", n), (double)((Integer)(output[2].Items[1])).LongValue, wUpper: 50, cUpper: 75);
n++; n++;
} }
n = 0; n = 0;
foreach (Sequence[] battery in batteries) foreach (Sequence[] battery in batteries)
{ {
checkState.WritePerformanceValue(String.Format("ups_battery_{0}_status", n), (double)((Integer)(battery[0].Items[1])).LongValue); node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_battery_{0}_status", n), (double)((Integer)(battery[0].Items[1])).LongValue);
checkState.WritePerformanceValue(String.Format("ups_battery_{0}_minutes_remain", n), (double)((Integer)(battery[1].Items[1])).LongValue, wLower: 20, cLower: 10 ); node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_battery_{0}_minutes_remain", n), (double)((Integer)(battery[1].Items[1])).LongValue, wLower: 20, cLower: 10 );
checkState.WritePerformanceValue(String.Format("ups_battery_{0}_capacity", n), (double)((Integer)(battery[2].Items[1])).LongValue, wLower: 75, cLower: 50); node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_battery_{0}_capacity", n), (double)((Integer)(battery[2].Items[1])).LongValue, wLower: 75, cLower: 50);
checkState.WritePerformanceValue(String.Format("ups_battery_{0}_voltage", n), (double)((Integer)(battery[3].Items[1])).LongValue / 10.0); node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_battery_{0}_voltage", n), (double)((Integer)(battery[3].Items[1])).LongValue / 10.0);
checkState.WritePerformanceValue(String.Format("ups_battery_{0}_temperature", n), (double)((Integer)(battery[4].Items[1])).LongValue, wUpper: 40, cUpper: 60); node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ups_battery_{0}_temperature", n), (double)((Integer)(battery[4].Items[1])).LongValue, wUpper: 40, cUpper: 60);
n++; n++;
} }

68
checks/CheckJob.cs
View File

@ -11,62 +11,27 @@ using System;
using ln.types.threads; using ln.types.threads;
using ln.skyscanner.entities; using ln.skyscanner.entities;
using ln.logging; using ln.logging;
using ln.types.odb; using ln.skyscanner.services;
using System.Linq;
using System.Collections.Generic;
namespace ln.skyscanner.checks namespace ln.skyscanner.checks
{ {
public class CheckJob : PoolJob public class CheckJob : PoolJob
{ {
public CheckService CheckService { get; }
public Node Node { get; } public Node Node { get; }
public SkyCheckState[] CheckStates public CheckJob(CheckService checkService,Node node)
{
get
{
return checkStates.Values.ToArray();
}
set
{
checkStates.Clear();
foreach (SkyCheckState checkState in value)
{
checkStates.Add(checkState.CheckName, checkState);
}
}
}
Dictionary<string, SkyCheckState> checkStates = new Dictionary<string, SkyCheckState>();
public CheckJob(Node node)
{ {
CheckService = checkService;
Name = String.Format("Interval check: {0} [{1}]",node.UniqueIdentity,node.PrimaryIP.ToString()); Name = String.Format("Interval check: {0} [{1}]",node.UniqueIdentity,node.PrimaryIP.ToString());
Node = node; Node = node;
} }
public override void Prepare()
{
/*
Query stateQuery = Query.Equals<SkyCheckState>("Node", Node.ID);
SkyCheckState[] skyCheckStates = SkyScanner.Instance.Entities.SkyCheckStates.Query(stateQuery).ToArray();
foreach (SkyCheckState checkState in skyCheckStates)
{
checkStates.Add(checkState.CheckName, checkState);
}
foreach (SkyCheck check in SkyCheck.SkyChecks)
if (!checkStates.ContainsKey(check.Name) && check.IsValid(Node))
{
checkStates.Add(check.Name, check.PrepareCheckState(SkyScanner.Instance.Checker, Node));
SkyScanner.Instance.Entities.SkyCheckStates.Insert(checkStates[check.Name]);
}
*/
}
public override void RunJob() public override void RunJob()
{ {
SkyCheck[] checks = SkyCheck.SkyChecks; SkyCheck[] checks = SkyCheck.SkyChecks;
long startTime = DateTimeOffset.Now.ToUnixTimeMilliseconds();
for (int n=0;n<checks.Length;n++) for (int n=0;n<checks.Length;n++)
{ {
setState("current check: {0}", checks[n].Name); setState("current check: {0}", checks[n].Name);
@ -76,18 +41,26 @@ namespace ln.skyscanner.checks
{ {
try try
{ {
SkyCheckState checkState = checkStates.ContainsKey(checks[n].Name) ? checkStates[checks[n].Name] : null; SkyCheckState checkState = null;
if (Node.ContainsCheckState(checks[n].Name))
{
checkState = Node.GetCheckState(checks[n].Name);
}
else
{
checkState = checks[n].PrepareCheckState(CheckService, Node);
Node.AddCheckState(checkState);
}
if (checkState != null) if (checkState != null)
{ {
checkState.LastCheckTime = DateTime.Now; checkState.LastCheckTime = DateTime.Now;
} }
checks[n].Check(SkyScanner.Instance.Checker, ref checkState, Node); checks[n].Check(CheckService, ref checkState, Node);
if (checkState != null) if (checkState != null)
{ {
checkState.CheckHistory(); checkState.CheckHistory();
SkyScanner.Instance.Checker.Save(checkState);
} }
} catch (Exception e) } catch (Exception e)
@ -103,6 +76,13 @@ namespace ln.skyscanner.checks
break; break;
} }
} }
long stopTime = DateTimeOffset.Now.ToUnixTimeMilliseconds();
if (CheckService.DEBUG)
Logging.Log(LogLevel.DEBUGDETAIL, "CheckTime: {0} ({1}): {2}ms", Name, Node.Name, (stopTime - startTime));
CheckService.EntityService.EnqueueUpsert(Node);
} }
public override int GetHashCode() public override int GetHashCode()

8
checks/Hostalive.cs
View File

@ -14,6 +14,8 @@ using ln.perfdb;
using System.Net.NetworkInformation; using System.Net.NetworkInformation;
using ln.logging; using ln.logging;
using ln.perfdb.storage; using ln.perfdb.storage;
using ln.skyscanner.services;
using ln.skyscanner.perfvalue;
namespace ln.skyscanner.checks namespace ln.skyscanner.checks
{ {
public class Hostalive : SkyCheck public class Hostalive : SkyCheck
@ -29,7 +31,7 @@ namespace ln.skyscanner.checks
public override bool IsCritical => true; public override bool IsCritical => true;
public override bool IsValid(Node node) => node.PrimaryIP != null; public override bool IsValid(Node node) => node.PrimaryIP != null;
public override void Check(SkyChecker skyChecker,ref SkyCheckState checkState,Node node) public override void Check(CheckService checkService, ref SkyCheckState checkState, Node node)
{ {
long roundTripTime = 0; long roundTripTime = 0;
int success = 0; int success = 0;
@ -47,11 +49,11 @@ namespace ln.skyscanner.checks
float fSuccess = (float)success / (float)n; float fSuccess = (float)success / (float)n;
checkState.WritePerformanceValue("replies", fSuccess, node.DeviceType == DeviceType.UNKNOWN ? Double.MinValue : 0.8, double.MaxValue, node.DeviceType == DeviceType.UNKNOWN ? Double.MinValue : 0.6, double.MaxValue); node.WritePerformanceValue(checkService.PerformanceValueService, Name, "replies", fSuccess, node.DeviceType == DeviceType.UNKNOWN ? Double.MinValue : 0.8, double.MaxValue, node.DeviceType == DeviceType.UNKNOWN ? Double.MinValue : 0.6, double.MaxValue);
if (success > 0) if (success > 0)
{ {
roundTripTime /= success; roundTripTime /= success;
checkState.WritePerformanceValue("rta", roundTripTime, 0, 80, 0, 140); node.WritePerformanceValue(checkService.PerformanceValueService, Name, "rta", roundTripTime, 0, 80, 0, 140);
} }
foreach (PerformanceValue pv in checkState.PerformanceValues) foreach (PerformanceValue pv in checkState.PerformanceValues)

8
checks/Mimosa.cs
View File

@ -15,7 +15,7 @@ namespace ln.skyscanner.checks
:base("mimosa") :base("mimosa")
{} {}
public override void Check(SkyChecker skyChecker, ref SkyCheckState checkState, Node node) public override void Check(CheckService checkService, ref SkyCheckState checkState, Node node)
{ {
MimosaCheckState mimosaCheckState = checkState as MimosaCheckState; MimosaCheckState mimosaCheckState = checkState as MimosaCheckState;
@ -23,7 +23,7 @@ namespace ln.skyscanner.checks
{ {
try try
{ {
using (SnmpInterface snmp = SnmpInterface.FromURI(snmpUri, skyChecker.SNMPEngine)) using (SnmpInterface snmp = SnmpInterface.FromURI(snmpUri, checkService.SNMPEngine))
{ {
Sequence[][] sigData = snmp.snmpWalk(new string[] { Sequence[][] sigData = snmp.snmpWalk(new string[] {
"1.3.6.1.4.1.43356.2.1.2.6.1.1.3", // Signal Level "1.3.6.1.4.1.43356.2.1.2.6.1.1.3", // Signal Level
@ -38,8 +38,8 @@ namespace ln.skyscanner.checks
double sig = (sigData[n][0].Items[1] as Integer).LongValue / 10.0; double sig = (sigData[n][0].Items[1] as Integer).LongValue / 10.0;
double snr = (sigData[n][1].Items[1] as Integer).LongValue / 10.0; double snr = (sigData[n][1].Items[1] as Integer).LongValue / 10.0;
mimosaCheckState.WritePerformanceValue(String.Format("ptp_rx_pwr_{0}",n), sig, -75.0, 0, -80.0, 0); node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ptp_rx_pwr_{0}",n), sig, -75.0, 0, -80.0, 0);
mimosaCheckState.WritePerformanceValue(String.Format("ptp_rx_snr_{0}",n), snr, 12, 99, 8, 99); node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ptp_rx_snr_{0}",n), snr, 12, 99, 8, 99);
} }

11
checks/SikluCheck.cs
View File

@ -4,6 +4,7 @@ using ln.logging;
using ln.types; using ln.types;
using ln.snmp; using ln.snmp;
using ln.snmp.types; using ln.snmp.types;
using ln.skyscanner.services;
namespace ln.skyscanner.checks namespace ln.skyscanner.checks
{ {
@ -14,13 +15,13 @@ namespace ln.skyscanner.checks
{ {
} }
public override void Check(SkyChecker skyChecker, ref SkyCheckState checkState, Node node) public override void Check(CheckService checkService, ref SkyCheckState checkState, Node node)
{ {
foreach (URI snmpUri in node.FindURIs("snmp")) foreach (URI snmpUri in node.FindURIs("snmp"))
{ {
try try
{ {
using (SnmpInterface snmp = SnmpInterface.FromURI(snmpUri, skyChecker.SNMPEngine)) using (SnmpInterface snmp = SnmpInterface.FromURI(snmpUri, checkService.SNMPEngine))
{ {
Sequence[][] ptp = snmp.snmpWalk(new string[] { Sequence[][] ptp = snmp.snmpWalk(new string[] {
"1.3.6.1.4.1.31926.2.1.1.19", // RX PWR "1.3.6.1.4.1.31926.2.1.1.19", // RX PWR
@ -31,9 +32,9 @@ namespace ln.skyscanner.checks
int n = 0; int n = 0;
foreach (Sequence[] row in ptp) foreach (Sequence[] row in ptp)
{ {
checkState.WritePerformanceValue(String.Format("ptp_rx_pwr_{0}", n), (double)((Integer)(row[0].Items[1])).LongValue, -75.0, 0, -80.0, 0); node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ptp_rx_pwr_{0}", n), (double)((Integer)(row[0].Items[1])).LongValue, -75.0, 0, -80.0, 0);
checkState.WritePerformanceValue(String.Format("ptp_tx_snr_{0}", n), (double)((Integer)(row[1].Items[1])).LongValue); node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ptp_tx_snr_{0}", n), (double)((Integer)(row[1].Items[1])).LongValue);
checkState.WritePerformanceValue(String.Format("ptp_tx_pwr_{0}", n), (double)((Integer)(row[2].Items[1])).LongValue); node.WritePerformanceValue(checkService.PerformanceValueService, Name, String.Format("ptp_tx_pwr_{0}", n), (double)((Integer)(row[2].Items[1])).LongValue);
/* long rxBytes = ((Integer)(row[4].Items[1])).LongValue; /* long rxBytes = ((Integer)(row[4].Items[1])).LongValue;
long txBytes = ((Integer)(row[5].Items[1])).LongValue; long txBytes = ((Integer)(row[5].Items[1])).LongValue;

2
checks/SkyCheck.cs
View File

@ -33,7 +33,7 @@ namespace ln.skyscanner.checks
public virtual bool IsCritical => false; public virtual bool IsCritical => false;
public abstract bool IsValid(Node node); public abstract bool IsValid(Node node);
public abstract void Check(SkyChecker skyChecker,ref SkyCheckState checkState,Node node); public abstract void Check(CheckService checkService,ref SkyCheckState checkState,Node node);
public virtual SkyCheckState PrepareCheckState(CheckService skyChecker,Node node) public virtual SkyCheckState PrepareCheckState(CheckService skyChecker,Node node)
{ {
return new SkyCheckState(this, node); return new SkyCheckState(this, node);

53
checks/SkyCheckState.cs
View File

@ -9,8 +9,10 @@
// **/ // **/
using System; using System;
using ln.skyscanner.entities; using ln.skyscanner.entities;
using ln.types.odb.attributes;
using System.Collections.Generic; using System.Collections.Generic;
using ln.skyscanner.perfvalue;
using ln.skyscanner.services;
using ln.json.attributes;
namespace ln.skyscanner.checks namespace ln.skyscanner.checks
{ {
public enum CheckState { OK, WARN, CRITICAL, FAIL, ERROR } public enum CheckState { OK, WARN, CRITICAL, FAIL, ERROR }
@ -29,12 +31,9 @@ namespace ln.skyscanner.checks
public class SkyCheckState public class SkyCheckState
{ {
[DocumentID]
public Guid ID; public Guid ID;
public readonly String CheckName; public readonly String CheckName;
[ByReference]
public readonly Node Node;
public PerformanceValue[] PerformanceValues => performanceValues.ToArray(); public PerformanceValue[] PerformanceValues => performanceValues.ToArray();
List<PerformanceValue> performanceValues = new List<PerformanceValue>(); List<PerformanceValue> performanceValues = new List<PerformanceValue>();
@ -69,29 +68,6 @@ namespace ln.skyscanner.checks
{ {
ID = Guid.NewGuid(); ID = Guid.NewGuid();
CheckName = skyCheck.Name; CheckName = skyCheck.Name;
Node = node;
}
public void WritePerformanceValue(String perfName,double value, double wLower = Double.MinValue, double wUpper = Double.MaxValue, double cLower = Double.MinValue, double cUpper = Double.MaxValue)
{
if (performanceValues == null)
performanceValues = new List<PerformanceValue>();
string[] perfPath = new string[] { Node.UniqueIdentity, CheckName, perfName };
perfName = string.Join("/", perfPath);
foreach (PerformanceValue performanceValue in performanceValues)
{
if (performanceValue.PerfName.Equals(perfName))
{
performanceValue.WritePerfValue(value);
return;
}
}
PerformanceValue newPerformanceValue = new PerformanceValue(perfPath, wLower, wUpper, cLower, cUpper);
performanceValues.Add(newPerformanceValue);
newPerformanceValue.WritePerfValue(value);
} }
public void CheckHistory() public void CheckHistory()
@ -102,6 +78,29 @@ namespace ln.skyscanner.checks
} }
} }
public void AddPerformanceValue(PerformanceValue performanceValue)
{
performanceValues.Add(performanceValue);
}
public PerformanceValue GetPerformanceValue(string perfName)
{
foreach (PerformanceValue performanceValue in performanceValues)
{
if (performanceValue.PerfName.Equals(perfName))
return performanceValue;
}
return null;
}
public bool ContainsPerformanceValue(string perfName)
{
foreach (PerformanceValue performanceValue in performanceValues)
{
if (performanceValue.PerfName.Equals(perfName))
return true;
}
return false;
}
} }

210
checks/SkyChecker.cs
View File

@ -1,210 +0,0 @@
// /**
// * File: SkyChecker.cs
// * Author: haraldwolff
// *
// * This file and it's content is copyrighted by the Author and / or copyright holder.
// * Any use wihtout proper permission is illegal and may lead to legal actions.
// *
// *
// **/
using System;
using System.IO;
using ln.types.btree;
using ln.perfdb.storage;
using ln.skyscanner.entities;
using ln.types.threads;
using System.Threading;
using System.Linq;
using ln.logging;
using ln.snmp;
using ln.http.resources;
using System.Collections.Generic;
namespace ln.skyscanner.checks
{
public class SkyChecker
{
public string BasePath { get; }
[Callable]
public PoolJob[] CurrentJobs => checkPool.CurrentPoolJobs;
[Callable]
public PoolJob[] QueuedJobs => checkPool.QueuedJobs;
[Callable]
public string[] PerfNames => perfFiles.Keys.ToArray();
public bool ContainsPerfFile(String perfName) => perfFiles.ContainsKey(perfName);
[Callable]
public SkyCheckState[] Issues
{
get
{
SkyCheckState[] checkStates = null;
lock (SkyScanner.Instance.Entities.SkyCheckStates)
{
checkStates = Timing.Meassure("scheduler: checkStates", () => SkyScanner.Instance.Entities.SkyCheckStates.ToArray());
}
return checkStates.Where((cs) => (!String.Empty.Equals(cs.Node?.Vendor) && ((cs.CheckState != CheckState.OK) || ((cs.UnchangedTime < TimeSpan.FromMinutes(10)) && (cs.History.Length > 1) )))).ToArray();
}
}
[Callable]
public SkyCheckState[] CheckStatesByHostID(Guid nodeID)
{
lock (SkyScanner.Instance.Entities.SkyCheckStates)
{
return SkyScanner.Instance.Entities.SkyCheckStates.Query("Node", nodeID).ToArray();
}
}
public SNMPEngine SNMPEngine { get; }
BTree<string, PerfFile> perfFiles = new BTree<string, PerfFile>();
Pool checkPool = new Pool(0);
Thread threadScheduler;
bool stopping;
List<SkyCheckState> saveQueue = new List<SkyCheckState>();
public SkyChecker()
{
BasePath = Path.Combine(SkyScanner.Instance.BasePath, "perfdb");
Logging.Log(LogLevel.INFO, "SkyChecker: created");
SNMPEngine = new SNMPEngine();
SNMPEngine.Timeout = 4000;
}
public void Start()
{
if ((threadScheduler != null) && (threadScheduler.IsAlive))
throw new NotSupportedException("SkyChecker.scheduler() already running");
checkPool.SetPoolSize(256);
Thread.Sleep(2500);
threadScheduler = new Thread(scheduler);
threadScheduler.Start();
}
public void Stop()
{
stopping = true;
if (threadScheduler!=null && threadScheduler.IsAlive)
{
threadScheduler.Interrupt();
}
checkPool.Abort();
checkPool.Close();
foreach (PerfFile perfFile in perfFiles.Values)
{
if (perfFile.IsOpen)
perfFile.Close();
}
perfFiles.Clear();
}
public PerfFile GetPerfFile(params string[] perfPath)
{
lock (this)
{
string perfName = string.Join("/", perfPath);
if (!perfFiles.ContainsKey(perfName))
{
String perfDirectory = Path.Combine(BasePath, Path.Combine(perfPath.Take(perfPath.Length - 1).ToArray()));
if (!Directory.Exists(perfDirectory))
Directory.CreateDirectory(perfDirectory);
String perfFileName = Path.Combine(perfDirectory, String.Format("{0}.perf", perfPath[perfPath.Length-1]));
PerfFile perfFile = new PerfFile(perfFileName);
perfFile.Open();
if (perfFile.FirstSection == null)
{
PerfFile.PerfFileSection s1 = new PerfFile.PerfFileSection(perfFile, null, TimeSpan.FromDays(28), 60, AggregationMethod.AVERAGE);
PerfFile.PerfFileSection s2 = new PerfFile.PerfFileSection(perfFile, s1, TimeSpan.FromDays(56), 300, AggregationMethod.AVERAGE);
PerfFile.PerfFileSection s3 = new PerfFile.PerfFileSection(perfFile, s2, TimeSpan.FromDays(84), 900, AggregationMethod.AVERAGE);
PerfFile.PerfFileSection s4 = new PerfFile.PerfFileSection(perfFile, s3, TimeSpan.FromDays(168), 1800, AggregationMethod.AVERAGE);
PerfFile.PerfFileSection s5 = new PerfFile.PerfFileSection(perfFile, s4, TimeSpan.FromDays(750), 3600, AggregationMethod.AVERAGE);
}
perfFile.Close();
perfFiles.Add(perfName, perfFile);
}
return perfFiles[perfName];
}
}
private void scheduler()
{
long nextMinute = DateTimeOffset.Now.ToUnixTimeMilliseconds();
try
{
while (!stopping)
{
List<CheckJob> checkJobs = new List<CheckJob>();
Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): scheduler save CheckStates");
lock (saveQueue)
{
foreach (SkyCheckState checkState in saveQueue)
{
SkyScanner.Instance.Entities.SkyCheckStates.Upsert(checkState);
}
saveQueue.Clear();
}
Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): scheduler starts");
foreach (Node node in SkyScanner.Instance.Entities.NodeCollection)
{
CheckJob checkJob = new CheckJob(node);
checkJobs.Add(checkJob);
}
Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): prepared {0} checks", checkJobs.Count);
Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): scheduled {0} checks", checkPool.Enqueue(checkJobs));
while ((nextMinute - DateTimeOffset.Now.ToUnixTimeMilliseconds()) < 0)
nextMinute += 60000;
Thread.Sleep((int)(nextMinute - DateTimeOffset.Now.ToUnixTimeMilliseconds()));
}
} catch (ThreadInterruptedException)
{
} catch (OperationCanceledException)
{
}
}
public void Save(SkyCheckState checkState)
{
lock (saveQueue)
{
saveQueue.Add(checkState);
}
}
public ComponentState State
{
get
{
return ComponentState.INITIALIZED;
}
}
}
}

20
checks/Ubiquiti.cs
View File

@ -17,13 +17,13 @@ namespace ln.skyscanner.checks
{ {
} }
public override void Check(SkyChecker skyChecker,ref SkyCheckState checkState,Node node) public override void Check(CheckService checkService,ref SkyCheckState checkState,Node node)
{ {
UbiquityCheckState ubiquityCheckState = checkState as UbiquityCheckState; UbiquityCheckState ubiquityCheckState = checkState as UbiquityCheckState;
foreach (URI snmpUri in node.FindURIs("snmp")) foreach (URI snmpUri in node.FindURIs("snmp"))
{ {
using (SnmpInterface snmp = SnmpInterface.FromURI(snmpUri,skyChecker.SNMPEngine)) using (SnmpInterface snmp = SnmpInterface.FromURI(snmpUri,checkService.SNMPEngine))
{ {
List<String> mibs = null; List<String> mibs = null;
try try
@ -32,7 +32,9 @@ namespace ln.skyscanner.checks
foreach (Sequence s in snmp.snmpWalk("1.3.6.1.2.1.1.9.1.2")) foreach (Sequence s in snmp.snmpWalk("1.3.6.1.2.1.1.9.1.2"))
{ {
Logging.Log(LogLevel.DEBUG, "Ubiquiti: snmpWalk over OIDs: {0}", (s.Items[1] as ObjectIdentifier).AsString); if (CheckService.DEBUG)
Logging.Log(LogLevel.DEBUG, "Ubiquiti: snmpWalk over OIDs: {0}", (s.Items[1] as ObjectIdentifier).AsString);
mibs.Add((s.Items[1] as ObjectIdentifier).AsString); mibs.Add((s.Items[1] as ObjectIdentifier).AsString);
} }
@ -53,10 +55,10 @@ namespace ln.skyscanner.checks
foreach (Sequence[] row in ptp) foreach (Sequence[] row in ptp)
{ {
ubiquityCheckState.WritePerformanceValue("ptp_rx_capa", (double)((Integer)(row[0].Items[1])).LongValue); node.WritePerformanceValue(checkService.PerformanceValueService, Name, "ptp_rx_capa", (double)((Integer)(row[0].Items[1])).LongValue);
ubiquityCheckState.WritePerformanceValue("ptp_tx_capa", (double)((Integer)(row[1].Items[1])).LongValue); node.WritePerformanceValue(checkService.PerformanceValueService, Name, "ptp_tx_capa", (double)((Integer)(row[1].Items[1])).LongValue);
ubiquityCheckState.WritePerformanceValue("ptp_rx_pwr", (double)((Integer)(row[2].Items[1])).LongValue, -75.0, 0, -80.0, 0); node.WritePerformanceValue(checkService.PerformanceValueService, Name, "ptp_rx_pwr", (double)((Integer)(row[2].Items[1])).LongValue, -75.0, 0, -80.0, 0);
ubiquityCheckState.WritePerformanceValue("ptp_tx_pwr", (double)((Integer)(row[3].Items[1])).LongValue); node.WritePerformanceValue(checkService.PerformanceValueService, Name, "ptp_tx_pwr", (double)((Integer)(row[3].Items[1])).LongValue);
long rxBytes = ((Integer)(row[4].Items[1])).LongValue; long rxBytes = ((Integer)(row[4].Items[1])).LongValue;
long txBytes = ((Integer)(row[5].Items[1])).LongValue; long txBytes = ((Integer)(row[5].Items[1])).LongValue;
@ -64,8 +66,8 @@ namespace ln.skyscanner.checks
ubiquityCheckState.RXRate.Update(rxBytes * 8); ubiquityCheckState.RXRate.Update(rxBytes * 8);
ubiquityCheckState.TXRate.Update(txBytes * 8); ubiquityCheckState.TXRate.Update(txBytes * 8);
ubiquityCheckState.WritePerformanceValue("ptp_rx_rate", ubiquityCheckState.RXRate.Current); node.WritePerformanceValue(checkService.PerformanceValueService, Name, "ptp_rx_rate", ubiquityCheckState.RXRate.Current);
ubiquityCheckState.WritePerformanceValue("ptp_tx_rate", ubiquityCheckState.TXRate.Current); node.WritePerformanceValue(checkService.PerformanceValueService, Name, "ptp_tx_rate", ubiquityCheckState.TXRate.Current);
} }
} }
} }

6
crawl/CrawledHost.cs
View File

@ -9,19 +9,13 @@
// **/ // **/
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using ln.types;
using System.Net;
using System.Linq; using System.Linq;
using ln.snmp.rfc1213;
using ln.types.odb;
using ln.types.net; using ln.types.net;
using ln.types.odb.attributes;
namespace ln.skyscanner.crawl namespace ln.skyscanner.crawl
{ {
public class CrawledHost public class CrawledHost
{ {
[DocumentID]
public readonly Guid ID = Guid.NewGuid(); public readonly Guid ID = Guid.NewGuid();
public String Name { get; set; } public String Name { get; set; }

7
crawl/CrawledSubnet.cs
View File

@ -8,17 +8,12 @@
// * // *
// **/ // **/
using System; using System;
using ln.types;
using ln.types.odb;
using System.Collections.Generic;
using System.Linq;
using ln.types.net; using ln.types.net;
using ln.types.odb.attributes;
namespace ln.skyscanner.crawl namespace ln.skyscanner.crawl
{ {
public class CrawledSubnet public class CrawledSubnet
{ {
[DocumentID]
Guid ID = Guid.NewGuid(); Guid ID = Guid.NewGuid();
public readonly Network4 Network; public readonly Network4 Network;

16
crawl/Crawler.cs
View File

@ -121,14 +121,14 @@ namespace ln.skyscanner.crawl
CrawlerState = ComponentState.STARTED; CrawlerState = ComponentState.STARTED;
if (SkyScanner.Instance.OptionCrawlVendor != null) //if (SkyScanner.Instance.OptionCrawlVendor != null)
{ //{
foreach (Node node in SkyScanner.Instance.Entities.NodeCollection.Query("Vendor", SkyScanner.Instance.OptionCrawlVendor)) // foreach (Node node in SkyScanner.Instance.Entities.NodeCollection.Query("Vendor", SkyScanner.Instance.OptionCrawlVendor))
{ // {
CrawledHost crawledHost = FindHostForIP(node.PrimaryIP); // CrawledHost crawledHost = FindHostForIP(node.PrimaryIP);
Crawl(crawledHost); // Crawl(crawledHost);
} // }
} //}
} }
} }

3
crawl/service/CrawlService.cs
View File

@ -9,7 +9,8 @@
// **/ // **/
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using ln.skyscanner.crawl.tests;
namespace ln.skyscanner.crawl.service namespace ln.skyscanner.crawl.service
{ {
public abstract class CrawlService public abstract class CrawlService

468
crawl/service/SNMP.cs
View File

@ -1,269 +1,269 @@
// /** //// /**
// * File: SNMP.cs //// * File: SNMP.cs
// * Author: haraldwolff //// * Author: haraldwolff
// * //// *
// * This file and it's content is copyrighted by the Author and / or copyright holder. //// * This file and it's content is copyrighted by the Author and / or copyright holder.
// * Any use wihtout proper permission is illegal and may lead to legal actions. //// * Any use wihtout proper permission is illegal and may lead to legal actions.
// * //// *
// * //// *
// **/ //// **/
using System; //using System;
using System.Net; //using System.Net;
using ln.snmp; //using ln.snmp;
using ln.snmp.endpoint; //using ln.snmp.endpoint;
using ln.types; //using ln.types;
using ln.snmp.types; //using ln.snmp.types;
using System.Runtime.Remoting.Messaging; //using System.Runtime.Remoting.Messaging;
using ln.skyscanner.crawl.service; //using System.Linq;
using System.Linq; //using ln.types.net;
using ln.types.net;
namespace ln.skyscanner.crawl.tests
{
public class SNMP : CrawlService
{
public string[] defaultCommunities = new string[0];
public SNMP(string[] defaultSecrets) //namespace ln.skyscanner.crawl.service
: base("snmp") //{
{ // public class SNMP : CrawlService
defaultCommunities = defaultSecrets; // {
} // public string[] defaultCommunities = new string[0];
public override bool Check(Crawl crawl) // public SNMP(string[] defaultSecrets)
{ // : base("snmp")
if (!crawl.CheckRequiredOption("ping")) // {
return false; // defaultCommunities = defaultSecrets;
// }
if (!TestCurrentHints(crawl.Host)) // public override bool Check(Crawl crawl)
{ // {
TestDefaults(crawl.Host); // if (!crawl.CheckRequiredOption("ping"))
} // return false;
return true; // if (!TestCurrentHints(crawl.Host))
} // {
// TestDefaults(crawl.Host);
// }
public override bool HostProvidesOption(Crawl crawl, params object[] parameters) // return true;
{ // }
return (crawl.Host.GetHint("snmp.version",null) != null) && ((crawl.Host.GetHint("snmp.username",null) != null) || (crawl.Host.GetHint("snmp.community",null) != null));
}
public bool TestCurrentHints(CrawledHost crawledHost) // public override bool HostProvidesOption(Crawl crawl, params object[] parameters)
{ // {
SnmpInterface snmpEndpoint = GetSnmpInterface(crawledHost); // return (crawl.Host.GetHint("snmp.version",null) != null) && ((crawl.Host.GetHint("snmp.username",null) != null) || (crawl.Host.GetHint("snmp.community",null) != null));
if (snmpEndpoint != null) // }
{
using (snmpEndpoint) // public bool TestCurrentHints(CrawledHost crawledHost)
{ // {
try // SnmpInterface snmpEndpoint = GetSnmpInterface(crawledHost);
{ // if (snmpEndpoint != null)
Variable prID = snmpEndpoint.snmpGet("1.3.6.1.2.1.1.2.0"); // {
} catch (SnmpError) // using (snmpEndpoint)
{ // {
return false; // try
} catch (TimeoutException) // {
{ // Variable prID = snmpEndpoint.snmpGet("1.3.6.1.2.1.1.2.0");
return false; // } catch (SnmpError)
} // {
return true; // return false;
} // } catch (TimeoutException)
} // {
return false; // return false;
} // }
// return true;
// }
// }
// return false;
// }
private bool TestSnmpV3(CrawledHost crawledHost) // private bool TestSnmpV3(CrawledHost crawledHost)
{ // {
IPv4[] ips = crawledHost.IPAddresses.Concat(new IPv4[] { crawledHost.PrimaryIP }).Distinct().ToArray(); // IPv4[] ips = crawledHost.IPAddresses.Concat(new IPv4[] { crawledHost.PrimaryIP }).Distinct().ToArray();
foreach (IPv4 ip in ips) // foreach (IPv4 ip in ips)
{ // {
using (USMEndpoint v3endpoint = new USMEndpoint(SNMPEngine.DefaultEngine, new IPEndPoint(ip, 161))) // using (USMEndpoint v3endpoint = new USMEndpoint(SNMPEngine.DefaultEngine, new IPEndPoint(ip, 161)))
{ // {
try // try
{ // {
v3endpoint.QueryEngineID(); // v3endpoint.QueryEngineID();
} // }
catch (TimeoutException) // catch (TimeoutException)
{ // {
} // }
if (v3endpoint.RemoteEngineID != null) // if (v3endpoint.RemoteEngineID != null)
{ // {
crawledHost.SetHint("snmp.version", 3); // crawledHost.SetHint("snmp.version", 3);
int c = 0; // int c = 0;
foreach (string community in defaultCommunities) // foreach (string community in defaultCommunities)
{ // {
c++; // c++;
v3endpoint.Username = "skytron"; // v3endpoint.Username = "skytron";
v3endpoint.AuthMethod = SnmpV3AuthMethod.SHA; // v3endpoint.AuthMethod = SnmpV3AuthMethod.SHA;
v3endpoint.AuthKeyPhrase = community; // v3endpoint.AuthKeyPhrase = community;
v3endpoint.LocalizeKeys(); // v3endpoint.LocalizeKeys();
try // try
{ // {
Variable prID = v3endpoint.snmpGet("1.3.6.1.2.1.1.2.0"); // Variable prID = v3endpoint.snmpGet("1.3.6.1.2.1.1.2.0");
crawledHost.SetHint("snmp.username", "skytron"); // crawledHost.SetHint("snmp.username", "skytron");
crawledHost.SetHint("snmp.authkey", community); // crawledHost.SetHint("snmp.authkey", community);
crawledHost.SetHint("snmp.address", ip); // crawledHost.SetHint("snmp.address", ip);
return true; // return true;
} // }
catch (TimeoutException) // catch (TimeoutException)
{ // {
} // }
catch (SnmpError) // catch (SnmpError)
{ // {
} // }
} // }
} // }
v3endpoint.Close(); // v3endpoint.Close();
} // }
} // }
return false; // return false;
} // }
private bool TestSnmpV2(CrawledHost crawledHost) // private bool TestSnmpV2(CrawledHost crawledHost)
{ // {
IPv4[] ips = crawledHost.IPAddresses.Concat(new IPv4[] { crawledHost.PrimaryIP }).Distinct().ToArray(); // IPv4[] ips = crawledHost.IPAddresses.Concat(new IPv4[] { crawledHost.PrimaryIP }).Distinct().ToArray();
foreach (IPv4 ip in ips) // foreach (IPv4 ip in ips)
{ // {
using (SnmpV2Endpoint v2endpoint = new SnmpV2Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(crawledHost.PrimaryIP, 161))) // using (SnmpV2Endpoint v2endpoint = new SnmpV2Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(crawledHost.PrimaryIP, 161)))
{ // {
foreach (String community in defaultCommunities) // foreach (String community in defaultCommunities)
{ // {
v2endpoint.CommunityString = community; // v2endpoint.CommunityString = community;
try // try
{ // {
Variable prID = v2endpoint.snmpGet("1.3.6.1.2.1.1.2.0"); // Variable prID = v2endpoint.snmpGet("1.3.6.1.2.1.1.2.0");
crawledHost.SetHint("snmp.version", 2); // crawledHost.SetHint("snmp.version", 2);
crawledHost.SetHint("snmp.community", community); // crawledHost.SetHint("snmp.community", community);
crawledHost.SetHint("snmp.address", ip); // crawledHost.SetHint("snmp.address", ip);
return true; // return true;
} // }
catch (SnmpError) // catch (SnmpError)
{ // {
} // }
catch (TimeoutException) // catch (TimeoutException)
{ // {
} // }
} // }
v2endpoint.Close(); // v2endpoint.Close();
} // }
} // }
return false; // return false;
} // }
private bool TestSnmpV1(CrawledHost crawledHost) // private bool TestSnmpV1(CrawledHost crawledHost)
{ // {
IPv4[] ips = crawledHost.IPAddresses.Concat(new IPv4[] { crawledHost.PrimaryIP }).Distinct().ToArray(); // IPv4[] ips = crawledHost.IPAddresses.Concat(new IPv4[] { crawledHost.PrimaryIP }).Distinct().ToArray();
foreach (IPv4 ip in ips) // foreach (IPv4 ip in ips)
{ // {
using (SnmpV1Endpoint v1endpoint = new SnmpV1Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(crawledHost.PrimaryIP, 161))) // using (SnmpV1Endpoint v1endpoint = new SnmpV1Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(crawledHost.PrimaryIP, 161)))
{ // {
foreach (String community in defaultCommunities) // foreach (String community in defaultCommunities)
{ // {
v1endpoint.CommunityString = community; // v1endpoint.CommunityString = community;
try // try
{ // {
Variable prID = v1endpoint.snmpGet("1.3.6.1.2.1.1.2.0"); // Variable prID = v1endpoint.snmpGet("1.3.6.1.2.1.1.2.0");
crawledHost.SetHint("snmp.version", 1); // crawledHost.SetHint("snmp.version", 1);
crawledHost.SetHint("snmp.community", community); // crawledHost.SetHint("snmp.community", community);
crawledHost.SetHint("snmp.address", ip); // crawledHost.SetHint("snmp.address", ip);
return true; // return true;
} // }
catch (SnmpError) // catch (SnmpError)
{ // {
} // }
catch (TimeoutException) // catch (TimeoutException)
{ // {
} // }
} // }
v1endpoint.Close(); // v1endpoint.Close();
} // }
} // }
return false; // return false;
} // }
public bool TestDefaults(CrawledHost crawledHost) // public bool TestDefaults(CrawledHost crawledHost)
{ // {
if (TestSnmpV3(crawledHost) ? true : TestSnmpV2(crawledHost) ? true : TestSnmpV1(crawledHost)) // if (TestSnmpV3(crawledHost) ? true : TestSnmpV2(crawledHost) ? true : TestSnmpV1(crawledHost))
{ // {
using (SnmpInterface snmp = GetSnmpInterface(crawledHost)) // using (SnmpInterface snmp = GetSnmpInterface(crawledHost))
{ // {
Variable prID = snmp.snmpGet("1.3.6.1.2.1.1.2.0"); // Variable prID = snmp.snmpGet("1.3.6.1.2.1.1.2.0");
crawledHost.SetHint("snmp.sysObjectID", (prID as ObjectIdentifier).AsString); // crawledHost.SetHint("snmp.sysObjectID", (prID as ObjectIdentifier).AsString);
try // try
{ // {
Sequence[] seqORids = snmp.snmpWalk("1.3.6.1.2.1.1.9.1.2").ToArray(); // Sequence[] seqORids = snmp.snmpWalk("1.3.6.1.2.1.1.9.1.2").ToArray();
string[] ORids = new string[seqORids.Length]; // string[] ORids = new string[seqORids.Length];
for (int n = 0; n < ORids.Length; n++) // for (int n = 0; n < ORids.Length; n++)
{ // {
ORids[n] = (seqORids[n].Items[1] as ObjectIdentifier).AsString; // ORids[n] = (seqORids[n].Items[1] as ObjectIdentifier).AsString;
} // }
crawledHost.SetHint("snmp.orids", ORids); // crawledHost.SetHint("snmp.orids", ORids);
} // }
catch (TimeoutException) // catch (TimeoutException)
{ } // { }
} // }
return true; // return true;
} // }
else // else
{ // {
crawledHost.SetHint("snmp.version", null); // crawledHost.SetHint("snmp.version", null);
crawledHost.SetHint("snmp.username", null); // crawledHost.SetHint("snmp.username", null);
crawledHost.SetHint("snmp.authkey", null); // crawledHost.SetHint("snmp.authkey", null);
crawledHost.SetHint("snmp.community", null); // crawledHost.SetHint("snmp.community", null);
crawledHost.SetHint("snmp.sysObjectID", null); // crawledHost.SetHint("snmp.sysObjectID", null);
return false; // return false;
} // }
} // }
public static SnmpInterface GetSnmpInterface(CrawledHost crawledHost) // public static SnmpInterface GetSnmpInterface(CrawledHost crawledHost)
{ // {
int snmpVersion = crawledHost.GetHint<int>("snmp.version", -1); // int snmpVersion = crawledHost.GetHint<int>("snmp.version", -1);
IPv4 snmpAddress = crawledHost.GetHint<IPv4>("snmp.address", crawledHost.PrimaryIP); // IPv4 snmpAddress = crawledHost.GetHint<IPv4>("snmp.address", crawledHost.PrimaryIP);
switch (snmpVersion) // switch (snmpVersion)
{ // {
case 1: // case 1:
SnmpV1Endpoint v1 = new SnmpV1Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(snmpAddress, 161)); // SnmpV1Endpoint v1 = new SnmpV1Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(snmpAddress, 161));
v1.CommunityString = crawledHost.GetHint<string>("snmp.community"); // v1.CommunityString = crawledHost.GetHint<string>("snmp.community");
return v1; // return v1;
case 2: // case 2:
SnmpV2Endpoint v2 = new SnmpV2Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(snmpAddress, 161)); // SnmpV2Endpoint v2 = new SnmpV2Endpoint(SNMPEngine.DefaultEngine, new IPEndPoint(snmpAddress, 161));
v2.CommunityString = crawledHost.GetHint<string>("snmp.community"); // v2.CommunityString = crawledHost.GetHint<string>("snmp.community");
return v2; // return v2;
case 3: // case 3:
if (crawledHost.GetHint<string>("snmp.username", null) != null) // if (crawledHost.GetHint<string>("snmp.username", null) != null)
{ // {
USMEndpoint endpoint = new USMEndpoint(SNMPEngine.DefaultEngine, new System.Net.IPEndPoint(snmpAddress, 161)); // USMEndpoint endpoint = new USMEndpoint(SNMPEngine.DefaultEngine, new System.Net.IPEndPoint(snmpAddress, 161));
endpoint.AuthMethod = SnmpV3AuthMethod.SHA; // endpoint.AuthMethod = SnmpV3AuthMethod.SHA;
endpoint.Username = crawledHost.GetHint<string>("snmp.username"); // endpoint.Username = crawledHost.GetHint<string>("snmp.username");
endpoint.AuthKeyPhrase = crawledHost.GetHint<string>("snmp.authkey"); // endpoint.AuthKeyPhrase = crawledHost.GetHint<string>("snmp.authkey");
return endpoint; // return endpoint;
} // }
return null; // return null;
default: // default:
return null; // return null;
} // }
} // }
} // }
} //}

216
crawl/service/SSH.cs
View File

@ -1,119 +1,119 @@
// /** //// /**
// * File: SSH.cs //// * File: SSH.cs
// * Author: haraldwolff //// * Author: haraldwolff
// * //// *
// * This file and it's content is copyrighted by the Author and / or copyright holder. //// * This file and it's content is copyrighted by the Author and / or copyright holder.
// * Any use wihtout proper permission is illegal and may lead to legal actions. //// * Any use wihtout proper permission is illegal and may lead to legal actions.
// * //// *
// * //// *
// **/ //// **/
using System; //using System;
using Renci.SshNet; //using Renci.SshNet;
using ln.types; //using ln.types;
using Renci.SshNet.Common; //using Renci.SshNet.Common;
using System.Net.Sockets; //using System.Net.Sockets;
using ln.types.net; //using ln.types.net;
using ln.skyscanner.crawl.service; //using ln.logging;
using ln.logging;
namespace ln.skyscanner.crawl.tests
{
public class SSH : CrawlService
{
public SSH()
: base("ssh")
{
}
public static bool CanConnect(CrawledHost crawledHost) //namespace ln.skyscanner.crawl.service
{ //{
int sshPort = crawledHost.GetHint<int>("ssh.port", -1); // public class SSH : CrawlService
IPv4 sshIP = crawledHost.GetHint<IPv4>("ssh.ip", null); // {
string sshUser = crawledHost.GetHint<string>("ssh.login", null); // public SSH()
string sshPassword = crawledHost.GetHint<string>("ssh.password", null); // : base("ssh")
// {
// }
if ((sshPort == -1) || !CanConnect(crawledHost,sshIP,sshPort,sshUser,sshPassword)) // public static bool CanConnect(CrawledHost crawledHost)
{ // {
if (!Scan(crawledHost)) // int sshPort = crawledHost.GetHint<int>("ssh.port", -1);
{ // IPv4 sshIP = crawledHost.GetHint<IPv4>("ssh.ip", null);
crawledHost.SetHint("ssh.port", -1); // string sshUser = crawledHost.GetHint<string>("ssh.login", null);
crawledHost.SetHint("ssh.ip", null); // string sshPassword = crawledHost.GetHint<string>("ssh.password", null);
crawledHost.SetHint("ssh.login", null);
crawledHost.SetHint("ssh.password", null);
crawledHost.SetHint("ssh.version", null);
return false;
}
}
return true;
}
private static bool Scan(CrawledHost crawledHost) // if ((sshPort == -1) || !CanConnect(crawledHost,sshIP,sshPort,sshUser,sshPassword))
{ // {
foreach (IPv4 ip in crawledHost.IPAddresses) // if (!Scan(crawledHost))
{ // {
foreach (int port in new int[] { 13022, 22 }) // crawledHost.SetHint("ssh.port", -1);
{ // crawledHost.SetHint("ssh.ip", null);
if (crawledHost.HasHint(String.Format("tcp.{0}",port)) && crawledHost.GetHint<bool>(String.Format("tcp.{0}", port))) // crawledHost.SetHint("ssh.login", null);
try // crawledHost.SetHint("ssh.password", null);
{ // crawledHost.SetHint("ssh.version", null);
foreach (Credential credential in SkyScanner.Instance.Crawler.Credentials) // return false;
{ // }
Logging.Log(LogLevel.DEBUG, "SSH trying {0}:{1}...", credential.Username, credential.Password.Substring(0, 4)); // }
// return true;
// }
if (CanConnect(crawledHost, ip, port, credential.Username, credential.Password, true)) // private static bool Scan(CrawledHost crawledHost)
return true; // {
} // foreach (IPv4 ip in crawledHost.IPAddresses)
} catch (SocketException) // {
{ // foreach (int port in new int[] { 13022, 22 })
continue; // {
} // if (crawledHost.HasHint(String.Format("tcp.{0}",port)) && crawledHost.GetHint<bool>(String.Format("tcp.{0}", port)))
} // try
} // {
return false; // foreach (Credential credential in SkyScanner.Instance.Crawler.Credentials)
} // {
// Logging.Log(LogLevel.DEBUG, "SSH trying {0}:{1}...", credential.Username, credential.Password.Substring(0, 4));
private static bool CanConnect(CrawledHost crawledHost, IPv4 host, int port, string username, string password,bool throwe = false) // if (CanConnect(crawledHost, ip, port, credential.Username, credential.Password, true))
{ // return true;
using (SshClient client = new SshClient(host.ToString(), port, username, password)) // }
{ // } catch (SocketException)
client.ConnectionInfo.Timeout = TimeSpan.FromSeconds(5); // {
try // continue;
{ // }
String authBanner = null; // }
client.ConnectionInfo.AuthenticationBanner += (object sender, AuthenticationBannerEventArgs e) => authBanner = e.BannerMessage; // }
// return false;
// }
client.Connect(); // private static bool CanConnect(CrawledHost crawledHost, IPv4 host, int port, string username, string password,bool throwe = false)
// {
// using (SshClient client = new SshClient(host.ToString(), port, username, password))
// {
// client.ConnectionInfo.Timeout = TimeSpan.FromSeconds(5);
// try
// {
// String authBanner = null;
// client.ConnectionInfo.AuthenticationBanner += (object sender, AuthenticationBannerEventArgs e) => authBanner = e.BannerMessage;
crawledHost.SetHint("ssh.port", client.ConnectionInfo.Port); // client.Connect();
crawledHost.SetHint("ssh.ip", host);
crawledHost.SetHint("ssh.login", client.ConnectionInfo.Username);
crawledHost.SetHint("ssh.password", password);
crawledHost.SetHint("ssh.version", client.ConnectionInfo.ServerVersion);
crawledHost.SetHint("ssh.authbanner", authBanner);
client.Disconnect(); // crawledHost.SetHint("ssh.port", client.ConnectionInfo.Port);
return true; // crawledHost.SetHint("ssh.ip", host);
} // crawledHost.SetHint("ssh.login", client.ConnectionInfo.Username);
catch (SshException sshe) // crawledHost.SetHint("ssh.password", password);
{ // crawledHost.SetHint("ssh.version", client.ConnectionInfo.ServerVersion);
Logging.Log(sshe); // crawledHost.SetHint("ssh.authbanner", authBanner);
}
catch (SocketException)
{
return false;
}
}
return false;
}
public override bool Check(Crawl crawl) // client.Disconnect();
{ // return true;
return CanConnect(crawl.Host); // }
// catch (SshException sshe)
// {
// Logging.Log(sshe);
// }
// catch (SocketException)
// {
// return false;
// }
// }
// return false;
// }
} // public override bool Check(Crawl crawl)
// {
// return CanConnect(crawl.Host);
public override bool HostProvidesOption(Crawl crawl, params object[] parameters) // }
{
return crawl.Host.GetHint<IPv4>("ssh.ip", null) != null; // public override bool HostProvidesOption(Crawl crawl, params object[] parameters)
} // {
} // return crawl.Host.GetHint<IPv4>("ssh.ip", null) != null;
} // }
// }
//}

146
crawl/service/TCP.cs
View File

@ -1,83 +1,83 @@
// /** //// /**
// * File: TCP.cs //// * File: TCP.cs
// * Author: haraldwolff //// * Author: haraldwolff
// * //// *
// * This file and it's content is copyrighted by the Author and / or copyright holder. //// * This file and it's content is copyrighted by the Author and / or copyright holder.
// * Any use wihtout proper permission is illegal and may lead to legal actions. //// * Any use wihtout proper permission is illegal and may lead to legal actions.
// * //// *
// * //// *
// **/ //// **/
using System; //using System;
using System.Net.Sockets; //using System.Net.Sockets;
namespace ln.skyscanner.crawl.service //namespace ln.skyscanner.crawl.service
{ //{
public class TCP : CrawlService // public class TCP : CrawlService
{ // {
public int[] Ports { get; } // public int[] Ports { get; }
public TCP(int[] ports) // public TCP(int[] ports)
:base("tcp") // :base("tcp")
{ // {
Ports = ports; // Ports = ports;
} // }
public override bool Check(Crawl crawl) // public override bool Check(Crawl crawl)
{ // {
if (!crawl.CheckRequiredOption("ping")) // if (!crawl.CheckRequiredOption("ping"))
return false; // return false;
int n = 0; // int n = 0;
foreach (int port in Ports) // foreach (int port in Ports)
{ // {
crawl.UpdateState(this, (double)n / (double)Ports.Length); // crawl.UpdateState(this, (double)n / (double)Ports.Length);
try // try
{ // {
using (TcpClient tcpClient = new TcpClient()) // using (TcpClient tcpClient = new TcpClient())
{ // {
IAsyncResult ar = tcpClient.BeginConnect(crawl.Host.PrimaryIP.ToString(), port, null, null); // IAsyncResult ar = tcpClient.BeginConnect(crawl.Host.PrimaryIP.ToString(), port, null, null);
System.Threading.WaitHandle wh = ar.AsyncWaitHandle; // System.Threading.WaitHandle wh = ar.AsyncWaitHandle;
try // try
{ // {
if (!ar.AsyncWaitHandle.WaitOne(TimeSpan.FromSeconds(1), false)) // if (!ar.AsyncWaitHandle.WaitOne(TimeSpan.FromSeconds(1), false))
{ // {
tcpClient.Close(); // tcpClient.Close();
throw new TimeoutException(); // throw new TimeoutException();
} // }
tcpClient.EndConnect(ar); // tcpClient.EndConnect(ar);
} // }
catch (Exception) // catch (Exception)
{ // {
throw; // throw;
} // }
tcpClient.Close(); // tcpClient.Close();
crawl.Host.SetHint(String.Format("tcp.{0}", port), true); // crawl.Host.SetHint(String.Format("tcp.{0}", port), true);
} // }
} // }
catch (Exception) // catch (Exception)
{ // {
crawl.Host.SetHint(String.Format("tcp.{0}", port), false); // crawl.Host.SetHint(String.Format("tcp.{0}", port), false);
} // }
} // }
return true; // return true;
} // }
public override bool HostProvidesOption(Crawl crawl,params object[] parameters) // public override bool HostProvidesOption(Crawl crawl,params object[] parameters)
{ // {
if (parameters.Length == 0) // if (parameters.Length == 0)
return false; // return false;
foreach (object oport in parameters) // foreach (object oport in parameters)
{ // {
int port = (int)oport; // int port = (int)oport;
if (!crawl.Host.GetHint(String.Format("tcp.{0}", port),false)) // if (!crawl.Host.GetHint(String.Format("tcp.{0}", port),false))
return false; // return false;
} // }
return true; // return true;
} // }
} // }
} //}

107
crawl/service/Ubiquity.cs
View File

@ -1,59 +1,58 @@
// /** //// /**
// * File: Ubiquity.cs //// * File: Ubiquity.cs
// * Author: haraldwolff //// * Author: haraldwolff
// * //// *
// * This file and it's content is copyrighted by the Author and / or copyright holder. //// * This file and it's content is copyrighted by the Author and / or copyright holder.
// * Any use wihtout proper permission is illegal and may lead to legal actions. //// * Any use wihtout proper permission is illegal and may lead to legal actions.
// * //// *
// * //// *
// **/ //// **/
using System; //using System;
using System.Linq; //using System.Linq;
using ln.snmp.endpoint; //using ln.snmp.endpoint;
using ln.skyscanner.crawl.tests; //using ln.snmp;
using ln.snmp; //using System.Collections.Generic;
using System.Collections.Generic; //using ln.snmp.types;
using ln.snmp.types; //namespace ln.skyscanner.crawl.service
namespace ln.skyscanner.crawl.service //{
{ // public class Ubiquity : CrawlService
public class Ubiquity : CrawlService // {
{ // public Ubiquity()
public Ubiquity() // : base("ubiquity")
: base("ubiquity") // {
{ // }
}
public override bool Check(Crawl crawl) // public override bool Check(Crawl crawl)
{ // {
if (crawl.CheckRequiredOption("snmp")) // if (crawl.CheckRequiredOption("snmp"))
{ // {
using (SnmpInterface snmp = SNMP.GetSnmpInterface(crawl.Host)) // using (SnmpInterface snmp = SNMP.GetSnmpInterface(crawl.Host))
{ // {
if (crawl.Host.GetHint("snmp.orids", new string[0]).Contains("1.3.6.1.4.1.41112")) // if (crawl.Host.GetHint("snmp.orids", new string[0]).Contains("1.3.6.1.4.1.41112"))
{ // {
List<Sequence> test = snmp.snmpWalk("1.3.6.1.4.1.41112.1.3.2.1.11"); // List<Sequence> test = snmp.snmpWalk("1.3.6.1.4.1.41112.1.3.2.1.11");
//List<Sequence> test2 = snmp.snmpWalk("1.3.6.1.4.1.41112.1.3.3.1.66"); // //List<Sequence> test2 = snmp.snmpWalk("1.3.6.1.4.1.41112.1.3.3.1.66");
if (test.Count > 0) // if (test.Count > 0)
crawl.Host.SetHint("ubiquity.ptp", true); // crawl.Host.SetHint("ubiquity.ptp", true);
else // else
crawl.Host.SetHint("ubiquity.ptp", false); // crawl.Host.SetHint("ubiquity.ptp", false);
test = snmp.snmpWalk("1.3.6.1.4.1.41112.1.4.7.1.1"); // test = snmp.snmpWalk("1.3.6.1.4.1.41112.1.4.7.1.1");
if (test.Count > 0) // if (test.Count > 0)
crawl.Host.SetHint("ubiquity.ptmp", true); // crawl.Host.SetHint("ubiquity.ptmp", true);
else // else
crawl.Host.SetHint("ubiquity.ptmp", false); // crawl.Host.SetHint("ubiquity.ptmp", false);
} // }
} // }
} // }
return false; // return false;
} // }
public override bool HostProvidesOption(Crawl crawl, params object[] parameters) // public override bool HostProvidesOption(Crawl crawl, params object[] parameters)
{ // {
return false; // return false;
} // }
} // }
} //}

3
entities/ConfiguredIP.cs
View File

@ -11,12 +11,11 @@ using System;
using ln.types.odb; using ln.types.odb;
using ln.types; using ln.types;
using ln.types.net; using ln.types.net;
using ln.types.odb.attributes;
namespace ln.skyscanner.entities namespace ln.skyscanner.entities
{ {
public class ConfiguredIP public class ConfiguredIP
{ {
[DocumentID]
public readonly Guid ID = Guid.NewGuid(); public readonly Guid ID = Guid.NewGuid();
public IPv4 IP { get; private set; } public IPv4 IP { get; private set; }

512
entities/GlobalNetwork.cs
View File

@ -1,303 +1,301 @@
using System; //using System;
using System.Collections.Generic; //using System.Collections.Generic;
using System.Collections; //using System.Collections;
using System.Security; //using System.Security;
using ln.types; //using ln.types;
using System.Linq; //using System.Linq;
using System.Threading; //using System.Threading;
using ln.skyscanner.crawl; //using ln.snmp.rfc1213;
using ln.snmp.rfc1213; //using ln.types.odb;
using ln.types.odb; //using ln.logging;
using ln.types.sync; //using System.Globalization;
using ln.logging; //using ln.types.threads;
using System.Globalization; //using ln.types.odb.values;
using ln.types.threads; //using ln.types.net;
using ln.types.odb.values; //using System.Runtime.Serialization.Formatters;
using ln.types.net;
using System.Runtime.Serialization.Formatters;
namespace ln.skyscanner.entities //namespace ln.skyscanner.entities
{ //{
public class GlobalNetwork : Persistent // //public class GlobalNetwork
{ // //{
public HopMap DefaultHopMap { get; private set; } = new HopMap(); // // public HopMap DefaultHopMap { get; private set; } = new HopMap();
private object _updateLock = new object(); // // private object _updateLock = new object();
public GlobalNetwork() // // public GlobalNetwork()
{ // // {
} // // }
public void EnsureSubnet(Network4 network) // // public void EnsureSubnet(Network4 network)
{ // // {
Subnet subnet = SkyScanner.Instance.Entities.SubnetCollection.Where(s => s.Network.Equals(network)).FirstOrDefault(); // // Subnet subnet = SkyScanner.Instance.Entities.SubnetCollection.Where(s => s.Network.Equals(network)).FirstOrDefault();
if (subnet == null) // // if (subnet == null)
{ // // {
subnet = new Subnet(network); // // subnet = new Subnet(network);
SkyScanner.Instance.Entities.SubnetCollection.Upsert(subnet); // // SkyScanner.Instance.Entities.SubnetCollection.Upsert(subnet);
} // // }
} // // }
public Node GetNode(Guid id) // // public Node GetNode(Guid id)
{ // // {
return SkyScanner.Instance.Entities.NodeCollection[id]; // // return SkyScanner.Instance.Entities.NodeCollection[id];
} // // }
public Subnet FindSubnetForIP(IPv4 ip) // // public Subnet FindSubnetForIP(IPv4 ip)
{ // // {
return SkyScanner.Instance.Entities.SubnetCollection.Where((net) => net.Network.Contains(ip)).FirstOrDefault(); // // return SkyScanner.Instance.Entities.SubnetCollection.Where((net) => net.Network.Contains(ip)).FirstOrDefault();
} // // }
public IEnumerable<Node> FindHostsInSubnet(Network4 network) // // public IEnumerable<Node> FindHostsInSubnet(Network4 network)
{ // // {
ODBValue firstIP = ODBMapper.Default.MapValue(network.First); // // ODBValue firstIP = ODBMapper.Default.MapValue(network.First);
ODBValue lastIP = ODBMapper.Default.MapValue(network.Last); // // ODBValue lastIP = ODBMapper.Default.MapValue(network.Last);
Query nodeByIpQuery = Query.OR( // // Query nodeByIpQuery = Query.OR(
Query.Equals<Node>("Interfaces[].ConfiguredIPs[].Network", network), // // Query.Equals<Node>("Interfaces[].ConfiguredIPs[].Network", network),
Query.IF<Node>("PrimaryIP", (ip) => ((firstIP <= ip) && (ip <= lastIP))) // // Query.IF<Node>("PrimaryIP", (ip) => ((firstIP <= ip) && (ip <= lastIP)))
); // // );
return SkyScanner.Instance.Entities.NodeCollection.Query(nodeByIpQuery); // // return SkyScanner.Instance.Entities.NodeCollection.Query(nodeByIpQuery);
} // // }
public IEnumerable<Node> FindNeighbors(Node node) // // public IEnumerable<Node> FindNeighbors(Node node)
{ // // {
HashSet<Network4> networks = new HashSet<Network4>(); // // HashSet<Network4> networks = new HashSet<Network4>();
HashSet<Node> nodes = new HashSet<Node>(); // // HashSet<Node> nodes = new HashSet<Node>();
if (node.Interfaces.Count > 0) // // if (node.Interfaces.Count > 0)
{ // // {
foreach (Network4 network in node.Networks) // // foreach (Network4 network in node.Networks)
networks.Add(network); // // networks.Add(network);
} // // }
Subnet subnet = FindSubnetForIP(node.PrimaryIP); // // Subnet subnet = FindSubnetForIP(node.PrimaryIP);
if (subnet != null) // // if (subnet != null)
networks.Add(subnet.Network); // // networks.Add(subnet.Network);
foreach (Network4 network in networks) // // foreach (Network4 network in networks)
foreach (Node neighbor in FindHostsInSubnet(network)) // // foreach (Node neighbor in FindHostsInSubnet(network))
nodes.Add(neighbor); // // nodes.Add(neighbor);
return nodes; // // return nodes;
} // // }
public Node FindNodeByIP(IEnumerable<IPv4> ips) // // public Node FindNodeByIP(IEnumerable<IPv4> ips)
{ // // {
foreach (IPv4 ip in ips) // // foreach (IPv4 ip in ips)
{ // // {
Node node = FindNodeByIP(ip); // // Node node = FindNodeByIP(ip);
if (node != null) // // if (node != null)
return node; // // return node;
} // // }
return null; // // return null;
} // // }
public Node FindNodeByIP(IPv4 ip) // // public Node FindNodeByIP(IPv4 ip)
{ // // {
Node node = SkyScanner.Instance.Entities.NodeCollection.Query("PrimaryIP", ip).FirstOrDefault(); // // Node node = SkyScanner.Instance.Entities.NodeCollection.Query("PrimaryIP", ip).FirstOrDefault();
if (node == null) // // if (node == null)
{ // // {
Query nodeByIpQuery = Query.Equals<Node>("Interfaces[].ConfiguredIPs[].IP", ip); // // Query nodeByIpQuery = Query.Equals<Node>("Interfaces[].ConfiguredIPs[].IP", ip);
node = SkyScanner.Instance.Entities.NodeCollection.Query(nodeByIpQuery).FirstOrDefault(); // // node = SkyScanner.Instance.Entities.NodeCollection.Query(nodeByIpQuery).FirstOrDefault();
} // // }
return node; // // return node;
} // // }
[Unsynced] // // [Unsynced]
private Queue<CrawledHost> updateQueue = new Queue<CrawledHost>(); // // private Queue<CrawledHost> updateQueue = new Queue<CrawledHost>();
public void EnqueueUpdate(CrawledHost crawledHost) // // public void EnqueueUpdate(CrawledHost crawledHost)
{ // // {
lock (updateQueue) // // lock (updateQueue)
{ // // {
if (!updateQueue.Contains(crawledHost)) // // if (!updateQueue.Contains(crawledHost))
updateQueue.Enqueue(crawledHost); // // updateQueue.Enqueue(crawledHost);
} // // }
} // // }
public void Update() // // public void Update()
{ // // {
lock (updateQueue) // // lock (updateQueue)
{ // // {
if (updateQueue.Count == 0) // // if (updateQueue.Count == 0)
return; // // return;
} // // }
while (true) // // while (true)
{ // // {
try // // try
{ // // {
CrawledHost crawledHost = null; // // CrawledHost crawledHost = null;
lock (updateQueue) // // lock (updateQueue)
{ // // {
if (updateQueue.Count > 0) // // if (updateQueue.Count > 0)
crawledHost = updateQueue.Dequeue(); // // crawledHost = updateQueue.Dequeue();
else // // else
break; // // break;
} // // }
Update(crawledHost); // // Update(crawledHost);
} catch (Exception e) // // } catch (Exception e)
{ // // {
Logging.Log(LogLevel.ERROR, "GlobalNetwork.Update(): Caught Exception: {0}",e); // // Logging.Log(LogLevel.ERROR, "GlobalNetwork.Update(): Caught Exception: {0}",e);
Logging.Log(e); // // Logging.Log(e);
} // // }
} // // }
} // // }
private void Update(CrawledHost crawledHost) // // private void Update(CrawledHost crawledHost)
{ // // {
Logging.Log(LogLevel.DEBUG, "Entity: update {0}", crawledHost.PrimaryIP); // // Logging.Log(LogLevel.DEBUG, "Entity: update {0}", crawledHost.PrimaryIP);
Node node = FindNodeByIP(crawledHost.PrimaryIP); // // Node node = FindNodeByIP(crawledHost.PrimaryIP);
if (node == null) // // if (node == null)
node = FindNodeByIP(crawledHost.IPAddresses); // // node = FindNodeByIP(crawledHost.IPAddresses);
if (node == null) // // if (node == null)
{ // // {
node = new Node(crawledHost.PrimaryIP); // // node = new Node(crawledHost.PrimaryIP);
node.Name = crawledHost.Name; // // node.Name = crawledHost.Name;
node.PrimaryMac = crawledHost.PrimaryHWAddr; // // node.PrimaryMac = crawledHost.PrimaryHWAddr;
SkyScanner.Instance.Entities.NodeCollection.Insert(node); // // SkyScanner.Instance.Entities.NodeCollection.Insert(node);
} // // }
if (!crawledHost.PrimaryIP.Equals(node.PrimaryIP)) // // if (!crawledHost.PrimaryIP.Equals(node.PrimaryIP))
{ // // {
Logging.Log(LogLevel.INFO,"GlobalNetwork.Update(): will not update Node with unmatched PrimaryIP {0} != {1}",node.PrimaryIP, crawledHost.PrimaryIP); // // Logging.Log(LogLevel.INFO,"GlobalNetwork.Update(): will not update Node with unmatched PrimaryIP {0} != {1}",node.PrimaryIP, crawledHost.PrimaryIP);
return; // // return;
} // // }
foreach (String si in crawledHost.GetHint<string[]>("rfc1213.interfaces",new string[0])) // // foreach (String si in crawledHost.GetHint<string[]>("rfc1213.interfaces",new string[0]))
{ // // {
String[] fields = si.Split(';'); // // String[] fields = si.Split(';');
NetworkInterface networkInterface = node.GetInterface(fields[0]); // // NetworkInterface networkInterface = node.GetInterface(fields[0]);
if (networkInterface == null) // // if (networkInterface == null)
{ // // {
networkInterface = new NetworkInterface(fields[0]); // // networkInterface = new NetworkInterface(fields[0]);
node.Interfaces.Add(networkInterface); // // node.Interfaces.Add(networkInterface);
} // // }
networkInterface.HWAddress = fields[1]; // // networkInterface.HWAddress = fields[1];
IPv4[][] crawledIPs = fields[2].Split(',').Where((sip) => !String.Empty.Equals(sip)) // // IPv4[][] crawledIPs = fields[2].Split(',').Where((sip) => !String.Empty.Equals(sip))
.Select((sip) => sip.Split('/').Select(ip => IPv4.Parse(ip)).ToArray()).ToArray(); // // .Select((sip) => sip.Split('/').Select(ip => IPv4.Parse(ip)).ToArray()).ToArray();
ConfiguredIP[] currentIPs = networkInterface.ConfiguredIPs.ToArray(); // // ConfiguredIP[] currentIPs = networkInterface.ConfiguredIPs.ToArray();
foreach (IPv4[] crawledIP in crawledIPs) // // foreach (IPv4[] crawledIP in crawledIPs)
{ // // {
int n; // // int n;
for (n=0; n < currentIPs.Length; n++) // // for (n=0; n < currentIPs.Length; n++)
if (currentIPs[n].IP.Equals(crawledIP[0])) // // if (currentIPs[n].IP.Equals(crawledIP[0]))
break; // // break;
if (n == currentIPs.Length) // // if (n == currentIPs.Length)
{ // // {
ConfiguredIP miss = new ConfiguredIP(crawledIP[0], new Network4(crawledIP[0], crawledIP[1])); // // ConfiguredIP miss = new ConfiguredIP(crawledIP[0], new Network4(crawledIP[0], crawledIP[1]));
networkInterface.ConfiguredIPs.Add(miss); // // networkInterface.ConfiguredIPs.Add(miss);
} // // }
} // // }
foreach (ConfiguredIP iip in currentIPs) // // foreach (ConfiguredIP iip in currentIPs)
{ // // {
IPv4 ip = iip.IP; // // IPv4 ip = iip.IP;
if (!crawledIPs.Select((ipl)=>ipl[0]).Contains(ip)) // // if (!crawledIPs.Select((ipl)=>ipl[0]).Contains(ip))
{ // // {
networkInterface.ConfiguredIPs.Remove(iip); // // networkInterface.ConfiguredIPs.Remove(iip);
} // // }
} // // }
} // // }
node.RemoveURI("ssh"); // // node.RemoveURI("ssh");
if (crawledHost.GetHint<int>("ssh.port", -1) != -1) // // if (crawledHost.GetHint<int>("ssh.port", -1) != -1)
{ // // {
node.AddURI( // // node.AddURI(
new URI( // // new URI(
String.Format("ssh://{0}:{1}@{2}:{3}", // // String.Format("ssh://{0}:{1}@{2}:{3}",
crawledHost.GetHint<string>("ssh.login"), // // crawledHost.GetHint<string>("ssh.login"),
crawledHost.GetHint<string>("ssh.password"), // // crawledHost.GetHint<string>("ssh.password"),
crawledHost.GetHint<IPv4>("ssh.ip"), // // crawledHost.GetHint<IPv4>("ssh.ip"),
crawledHost.GetHint<int>("ssh.port") // // crawledHost.GetHint<int>("ssh.port")
) // // )
) // // )
); // // );
} // // }
node.RemoveURI("snmp"); // // node.RemoveURI("snmp");
if (crawledHost.GetHint("snmp.version", -1) == 3) // // if (crawledHost.GetHint("snmp.version", -1) == 3)
{ // // {
node.AddURI(new URI(String.Format("snmp://{0}:{1}:{2}@{3}#{4}", // // node.AddURI(new URI(String.Format("snmp://{0}:{1}:{2}@{3}#{4}",
crawledHost.GetHint("snmp.username",""), // // crawledHost.GetHint("snmp.username",""),
crawledHost.GetHint("snmp.authkey",""), // // crawledHost.GetHint("snmp.authkey",""),
crawledHost.GetHint("snmp.privkey",""), // // crawledHost.GetHint("snmp.privkey",""),
crawledHost.GetHint<IPv4>("snmp.address"), // // crawledHost.GetHint<IPv4>("snmp.address"),
crawledHost.GetHint<int>("snmp.version") // // crawledHost.GetHint<int>("snmp.version")
))); // // )));
} // // }
else if (crawledHost.GetHint("snmp.version", -1) > 0) // // else if (crawledHost.GetHint("snmp.version", -1) > 0)
{ // // {
node.AddURI(new URI(String.Format("snmp://{0}@{1}#{2}", // // node.AddURI(new URI(String.Format("snmp://{0}@{1}#{2}",
crawledHost.GetHint("snmp.community", ""), // // crawledHost.GetHint("snmp.community", ""),
crawledHost.GetHint<IPv4>("snmp.address"), // // crawledHost.GetHint<IPv4>("snmp.address"),
crawledHost.GetHint<int>("snmp.version") // // crawledHost.GetHint<int>("snmp.version")
))); // // )));
} // // }
node.RemoveURI("http"); // // node.RemoveURI("http");
if (crawledHost.GetHint("http.port",-1) != -1) // // if (crawledHost.GetHint("http.port",-1) != -1)
{ // // {
node.AddURI(new URI(String.Format("http://{0}:{1}", // // node.AddURI(new URI(String.Format("http://{0}:{1}",
crawledHost.GetHint<IPv4>("http.ip", null), // // crawledHost.GetHint<IPv4>("http.ip", null),
crawledHost.GetHint<int>("http.port")) // // crawledHost.GetHint<int>("http.port"))
)); // // ));
} // // }
string httpIndex = crawledHost.GetHint("http.content", ""); // // string httpIndex = crawledHost.GetHint("http.content", "");
if (httpIndex.Contains("<title>RouterOS router configuration page</title>")) // // if (httpIndex.Contains("<title>RouterOS router configuration page</title>"))
{ // // {
node.Vendor = "MicroTik"; // // node.Vendor = "MicroTik";
} else if (httpIndex.Contains("mimosa-white-web-logo.png")) // // } else if (httpIndex.Contains("mimosa-white-web-logo.png"))
{ // // {
node.Vendor = "Mimosa"; // // node.Vendor = "Mimosa";
if (httpIndex.Contains("<title>B5c</title>")) // // if (httpIndex.Contains("<title>B5c</title>"))
{ // // {
node.Product = "B5c"; // // node.Product = "B5c";
node.DeviceType = DeviceType.PTP; // // node.DeviceType = DeviceType.PTP;
} // // }
} // // }
if (crawledHost.GetHint("http.server", "").Equals("Viprinet")) // // if (crawledHost.GetHint("http.server", "").Equals("Viprinet"))
node.Vendor = "Viprinet"; // // node.Vendor = "Viprinet";
if (crawledHost.GetHint("snmp.sysObjectID", "").Equals("1.3.6.1.4.1.318.1.3.27")) // // if (crawledHost.GetHint("snmp.sysObjectID", "").Equals("1.3.6.1.4.1.318.1.3.27"))
{ // // {
node.Vendor = "APC"; // // node.Vendor = "APC";
node.DeviceType = DeviceType.UPS; // // node.DeviceType = DeviceType.UPS;
} else if (crawledHost.GetHint("snmp.orids", new string[0]).Contains("1.3.6.1.4.1.41112")) // // } else if (crawledHost.GetHint("snmp.orids", new string[0]).Contains("1.3.6.1.4.1.41112"))
{ // // {
node.Vendor = "Ubiquity"; // // node.Vendor = "Ubiquity";
} // // }
if (crawledHost.GetHint<bool>("ubiquity.ptp", false)) // // if (crawledHost.GetHint<bool>("ubiquity.ptp", false))
{ // // {
node.DeviceType = DeviceType.PTP; // // node.DeviceType = DeviceType.PTP;
} else if (crawledHost.GetHint<bool>("ubiquity.ptmp", false)) // // } else if (crawledHost.GetHint<bool>("ubiquity.ptmp", false))
{ // // {
node.DeviceType = DeviceType.PTMP; // // node.DeviceType = DeviceType.PTMP;
} // // }
foreach (Network4 network in node.Networks) // // foreach (Network4 network in node.Networks)
EnsureSubnet(network); // // EnsureSubnet(network);
node.LastUpdate = DateTime.Now; // // node.LastUpdate = DateTime.Now;
SkyScanner.Instance.Entities.NodeCollection.Upsert(node); // // SkyScanner.Instance.Entities.NodeCollection.Upsert(node);
} // // }
} // //}
} //}

300
entities/HopMap.cs
View File

@ -15,185 +15,185 @@ using ln.types.btree;
using ln.types.net; using ln.types.net;
namespace ln.skyscanner.entities namespace ln.skyscanner.entities
{ {
public class HopMap : Persistent //public class HopMap : Persistent
{ //{
public Node TopNode { get; private set; } // public Node TopNode { get; private set; }
public GlobalNetwork GlobalNetwork => SkyScanner.Instance.Entities.GlobalNetwork; // public GlobalNetwork GlobalNetwork => SkyScanner.Instance.Entities.GlobalNetwork;
public int MaxDepth { get; set; } = 16; // public int MaxDepth { get; set; } = 16;
private BTree<Node, HopNode> hopNodes = new BTree<Node, HopNode>(CompareNodes); // private BTree<Node, HopNode> hopNodes = new BTree<Node, HopNode>(CompareNodes);
private HashSet<Network4> networks = new HashSet<Network4>(); // private HashSet<Network4> networks = new HashSet<Network4>();
public HopMap() // public HopMap()
{ // {
} // }
public HopMap(Node node) // public HopMap(Node node)
{ // {
Reset(node); // Reset(node);
} // }
public void Reset(Node node) // public void Reset(Node node)
{ // {
hopNodes.Clear(); // hopNodes.Clear();
TopNode = node; // TopNode = node;
HopNode hopNode = new HopNode(this, node); // HopNode hopNode = new HopNode(this, node);
} // }
public int GetHopCount(Node node) // public int GetHopCount(Node node)
{ // {
if (!hopNodes.ContainsKey(node)) // if (!hopNodes.ContainsKey(node))
return -1; // return -1;
return hopNodes[node].HopCount; // return hopNodes[node].HopCount;
} // }
public HopNode[] HopNodes // public HopNode[] HopNodes
{ // {
get // get
{ // {
return hopNodes.Values.ToArray(); // return hopNodes.Values.ToArray();
} // }
} // }
public class HopNode : IComparable // public class HopNode : IComparable
{ // {
public HopMap HopMap { get; } // public HopMap HopMap { get; }
public HopNode Parent { get; private set; } // public HopNode Parent { get; private set; }
public Node Node; // public Node Node;
public HopNode[] Peers => peers.ToArray(); // public HopNode[] Peers => peers.ToArray();
private HashSet<HopNode> peers = new HashSet<HopNode>(); // private HashSet<HopNode> peers = new HashSet<HopNode>();
private int hopCount = 0; // private int hopCount = 0;
public HopNode(HopMap hopMap, Node node) // public HopNode(HopMap hopMap, Node node)
{ // {
HopMap = hopMap; // HopMap = hopMap;
Node = node; // Node = node;
HopMap.hopNodes.Add(node, this); // HopMap.hopNodes.Add(node, this);
FindPeers(); // FindPeers();
} // }
public HopNode(HopNode parent, Node node) // public HopNode(HopNode parent, Node node)
{ // {
HopMap = parent.HopMap; // HopMap = parent.HopMap;
Node = node; // Node = node;
Attach(parent); // Attach(parent);
FindPeers(); // FindPeers();
} // }
private void FindPeers() // private void FindPeers()
{ // {
foreach (Network4 network in Node.Networks) // foreach (Network4 network in Node.Networks)
{ // {
EnsureNetwork(network); // EnsureNetwork(network);
} // }
EnsureNetwork(HopMap.GlobalNetwork.FindSubnetForIP(Node.PrimaryIP).Network); // EnsureNetwork(HopMap.GlobalNetwork.FindSubnetForIP(Node.PrimaryIP).Network);
} // }
private void EnsureNetwork(Network4 network) // private void EnsureNetwork(Network4 network)
{ // {
if (!HopMap.networks.Contains(network)) // if (!HopMap.networks.Contains(network))
{ // {
HopMap.networks.Add(network); // HopMap.networks.Add(network);
FindPeers(network); // FindPeers(network);
} // }
} // }
private void FindPeers(Network4 network) // private void FindPeers(Network4 network)
{ // {
foreach (Node peerNode in HopMap.GlobalNetwork.FindHostsInSubnet(network)) // foreach (Node peerNode in HopMap.GlobalNetwork.FindHostsInSubnet(network))
{ // {
if (!HopMap.hopNodes.ContainsKey(peerNode)) // if (!HopMap.hopNodes.ContainsKey(peerNode))
{ // {
new HopNode(this, peerNode); // new HopNode(this, peerNode);
} // }
else // else
{ // {
HopNode existingHopNode = HopMap.hopNodes[peerNode]; // HopNode existingHopNode = HopMap.hopNodes[peerNode];
if (existingHopNode.HopCount > (HopCount + 1)) // if (existingHopNode.HopCount > (HopCount + 1))
{ // {
existingHopNode.Detach(); // existingHopNode.Detach();
existingHopNode.Attach(this); // existingHopNode.Attach(this);
} // }
} // }
} // }
} // }
public int HopCount // public int HopCount
{ // {
get => hopCount; // get => hopCount;
set // set
{ // {
hopCount = value; // hopCount = value;
foreach (HopNode peer in peers) // foreach (HopNode peer in peers)
{ // {
peer.HopCount = hopCount + 1; // peer.HopCount = hopCount + 1;
} // }
} // }
} // }
public void Attach(HopNode parent) // public void Attach(HopNode parent)
{ // {
Detach(); // Detach();
Parent = parent; // Parent = parent;
if (!Object.ReferenceEquals(null, Parent)) // if (!Object.ReferenceEquals(null, Parent))
{ // {
HopMap.hopNodes.Add(this.Node, this); // HopMap.hopNodes.Add(this.Node, this);
Parent.peers.Add(this); // Parent.peers.Add(this);
HopCount = parent.hopCount + 1; // HopCount = parent.hopCount + 1;
} // }
else // else
{ // {
HopCount = 0; // HopCount = 0;
} // }
} // }
public void Detach() // public void Detach()
{ // {
if (!Object.ReferenceEquals(null, Parent)) // if (!Object.ReferenceEquals(null, Parent))
{ // {
Parent.peers.Remove(this); // Parent.peers.Remove(this);
Parent = null; // Parent = null;
HopMap.hopNodes.Remove(this.Node); // HopMap.hopNodes.Remove(this.Node);
} // }
} // }
public int CompareTo(object obj) // public int CompareTo(object obj)
{ // {
if (obj is HopNode) // if (obj is HopNode)
{ // {
HopNode you = obj as HopNode; // HopNode you = obj as HopNode;
return Node.ID.CompareTo(you.Node.ID); // return Node.ID.CompareTo(you.Node.ID);
} // }
return this.GetHashCode() - obj.GetHashCode(); // return this.GetHashCode() - obj.GetHashCode();
} // }
public IPv4[] Path => _path().ToArray(); // public IPv4[] Path => _path().ToArray();
private List<IPv4> _path() // private List<IPv4> _path()
{ // {
if (Parent == null) // if (Parent == null)
return new List<IPv4>(new IPv4[] { Node.PrimaryIP }); // return new List<IPv4>(new IPv4[] { Node.PrimaryIP });
List<IPv4> l = Parent._path(); // List<IPv4> l = Parent._path();
l.Add(Node.PrimaryIP); // l.Add(Node.PrimaryIP);
return l; // return l;
} // }
} // }
private static int CompareNodes(Node a, Node b) // private static int CompareNodes(Node a, Node b)
{ // {
return a.ID.CompareTo(b.ID); // return a.ID.CompareTo(b.ID);
} // }
} //}
} }

2
entities/L2Segment.cs
View File

@ -1,12 +1,10 @@
using System; using System;
using ln.types.odb.attributes;
using System.Collections.Generic; using System.Collections.Generic;
using ln.types.net; using ln.types.net;
namespace ln.skyscanner.entities namespace ln.skyscanner.entities
{ {
public class L2Segment public class L2Segment
{ {
[DocumentID]
public Guid ID = Guid.NewGuid(); public Guid ID = Guid.NewGuid();
public String Name; public String Name;

6
entities/NetworkInterface.cs
View File

@ -9,17 +9,11 @@
// **/ // **/
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using ln.types;
using ln.skyscanner.crawl;
using ln.types.odb;
using ln.logging;
using ln.types.net; using ln.types.net;
using ln.types.odb.attributes;
namespace ln.skyscanner.entities namespace ln.skyscanner.entities
{ {
public class NetworkInterface public class NetworkInterface
{ {
[DocumentID]
public readonly Guid ID = Guid.NewGuid(); public readonly Guid ID = Guid.NewGuid();
public string Name { get; private set; } = ""; public string Name { get; private set; } = "";

50
entities/Node.cs
View File

@ -14,12 +14,14 @@ using System.Linq;
using ln.types.odb; using ln.types.odb;
using ln.types.threads; using ln.types.threads;
using ln.types.net; using ln.types.net;
using ln.types.odb.attributes; using ln.skyscanner.checks;
using ln.skyscanner.perfvalue;
using ln.json.attributes;
using ln.skyscanner.services;
namespace ln.skyscanner.entities namespace ln.skyscanner.entities
{ {
public class Node public class Node
{ {
[DocumentID]
public Guid ID { get; set; } = Guid.NewGuid(); public Guid ID { get; set; } = Guid.NewGuid();
public IPv4 PrimaryIP { get; set; } public IPv4 PrimaryIP { get; set; }
@ -42,10 +44,14 @@ namespace ln.skyscanner.entities
public DeviceType DeviceType { get; set; } public DeviceType DeviceType { get; set; }
[JSONMapping(Private = true)]
public List<NetworkInterface> Interfaces { get; private set; } = new List<NetworkInterface>(); public List<NetworkInterface> Interfaces { get; private set; } = new List<NetworkInterface>();
[JSONMapping(Private = true)]
public IEnumerable<IPv4> IPAdresses => Interfaces.SelectMany(intf => intf.ConfiguredIPs.Select(nip => nip.IP)); public IEnumerable<IPv4> IPAdresses => Interfaces.SelectMany(intf => intf.ConfiguredIPs.Select(nip => nip.IP));
[JSONMapping(Private = true)]
public IEnumerable<Network4> Networks => Interfaces.SelectMany(intf => intf.ConfiguredIPs.Select(nip => nip.Network)).Distinct(); public IEnumerable<Network4> Networks => Interfaces.SelectMany(intf => intf.ConfiguredIPs.Select(nip => nip.Network)).Distinct();
[JSONMapping(Private = true)]
public IEnumerable<Subnet> Subnets => SkyScanner.Instance.Entities.SubnetCollection.Query(Query.Equals<Network4>("Network", Networks.Select(net => net.Network))); public IEnumerable<Subnet> Subnets => SkyScanner.Instance.Entities.SubnetCollection.Query(Query.Equals<Network4>("Network", Networks.Select(net => net.Network)));
private string uniqueIdentity; private string uniqueIdentity;
@ -62,6 +68,14 @@ namespace ln.skyscanner.entities
private HashSet<URI> uris = new HashSet<URI>(); private HashSet<URI> uris = new HashSet<URI>();
private HashSet<string> checks = new HashSet<string>(); private HashSet<string> checks = new HashSet<string>();
private Dictionary<string, SkyCheckState> checkStates = new Dictionary<string, SkyCheckState>();
public SkyCheckState[] CheckStates => checkStates.Values.ToArray();
public string[] CheckStateNames => checkStates.Keys.ToArray();
//private Dictionary<string, PerformanceValue> performanceValues = new Dictionary<string, PerformanceValue>();
//public PerformanceValue[] PerformanceValues => performanceValues.Values.ToArray();
//public string[] PerformanceValueNames => performanceValues.Keys.ToArray();
private Node() private Node()
{ {
} }
@ -118,6 +132,38 @@ namespace ln.skyscanner.entities
return GetInterface(intfName) != null; return GetInterface(intfName) != null;
} }
public bool ContainsCheckState(string name) => checkStates.ContainsKey(name);
public SkyCheckState GetCheckState(string name) => checkStates[name];
public void AddCheckState(SkyCheckState checkState)
{
checkStates[checkState.CheckName] = checkState;
}
//public bool ContainsPerformanceValue(String name) => performanceValues.ContainsKey(name);
//public PerformanceValue GetPerformanceValue(string name) => performanceValues[name];
//public void AddPerformanceValue(PerformanceValue performanceValue)
//{
// performanceValues[Name] = performanceValue;
//}
public void WritePerformanceValue(PerformanceValueService performanceValueService, String checkName, String perfName, double value, double wLower = Double.MinValue, double wUpper = Double.MaxValue, double cLower = Double.MinValue, double cUpper = Double.MaxValue)
{
string[] perfPath = new string[] { UniqueIdentity, checkName, perfName };
perfName = string.Join("/", perfPath);
SkyCheckState skyCheckState = GetCheckState(checkName);
PerformanceValue performanceValue = skyCheckState.GetPerformanceValue(perfName);
if (performanceValue == null)
{
performanceValue = new PerformanceValue(perfPath, wLower, wUpper, cLower, cUpper);
skyCheckState.AddPerformanceValue(performanceValue);
}
performanceValueService.WriteValue(performanceValue, value);
}
public override int GetHashCode() public override int GetHashCode()
{ {
return ID.GetHashCode(); return ID.GetHashCode();

11
entities/PointOfPresence.cs
View File

@ -12,20 +12,15 @@ using ln.types.odb;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using ln.types; using ln.types;
using ln.types.odb.attributes;
using ln.types.odb.collections;
namespace ln.skyscanner.entities namespace ln.skyscanner.entities
{ {
public class PointOfPresence public class PointOfPresence
{ {
[DocumentID]
public Guid ID = Guid.NewGuid(); public Guid ID = Guid.NewGuid();
public String Name { get; set; } public String Name { get; set; }
public String ForeignName { get; set; } public String ForeignName { get; set; }
public RefList<Node> Nodes { get; private set; } = new RefList<Node>();
public GeoLocation GeoLocation { get; set; } public GeoLocation GeoLocation { get; set; }
public DateTime Created { get; private set; } = DateTime.Now; public DateTime Created { get; private set; } = DateTime.Now;
@ -33,7 +28,7 @@ namespace ln.skyscanner.entities
{ {
get get
{ {
return SkyScanner.Instance.Entities.L2SegmentCollection.Query("PoPs[]", this); throw new NotImplementedException();
} }
} }
@ -41,10 +36,6 @@ namespace ln.skyscanner.entities
{ {
} }
public void AddNode(Node node) => Nodes.Add(node);
public void RemoveNode(Node node) => Nodes.Remove(node);
} }
} }

6
entities/Subnet.cs
View File

@ -13,14 +13,10 @@ using ln.types.odb;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using ln.types.net; using ln.types.net;
using ln.types.odb.attributes;
namespace ln.skyscanner.entities namespace ln.skyscanner.entities
{ {
public class Subnet : Persistent public class Subnet
{ {
public SkyEntities SkyEntities => SkyScanner.Instance.Entities;
[DocumentID]
public Guid ID { get; private set; } = Guid.NewGuid(); public Guid ID { get; private set; } = Guid.NewGuid();
public readonly Network4 Network; public readonly Network4 Network;

130
http/CheckerApi.cs
View File

@ -1,130 +0,0 @@
// /**
// * File: CheckerApi.cs
// * Author: haraldwolff
// *
// * This file and it's content is copyrighted by the Author and / or copyright holder.
// * Any use wihtout proper permission is illegal and may lead to legal actions.
// *
// *
// **/
using System;
using System.Collections.Generic;
using ln.http;
using ln.http.resources;
using ln.types.threads;
using ln.perfdb.storage;
using System.Net;
using ln.skyscanner.checks;
using ln.types.odb;
using System.Linq;
using ln.json.mapping;
namespace ln.skyscanner.http
{
public class CheckerApi : JsonCallResource
{
public CheckerApi(Resource container)
: base(container, "checker")
{
new Checks(this);
}
[Callable]
public PoolJob[] CurrentPoolJobs => SkyScanner.Instance.Checker.CurrentJobs;
[Callable]
public PoolJob[] QueuedPoolJobs => SkyScanner.Instance.Checker.QueuedJobs;
[Callable]
public SkyCheckState[] Issues
{
get
{
Query issueQuery = Query.EqualsNot<SkyCheckState>("CheckState", CheckState.OK);
return SkyScanner.Instance.Entities.SkyCheckStates.Query(issueQuery).ToArray();
}
}
class Checks : BaseResource
{
public Checks(CheckerApi container)
:base(container,"checks")
{ }
public override bool HandlesDispatching => true;
public override bool Contains(string name)
{
return base.Contains(name) || SkyScanner.Instance.Checker.ContainsPerfFile(name);
}
public override Resource GetResource(string name)
{
if (base.Contains(name))
return base.GetResource(name);
if (SkyScanner.Instance.Checker.ContainsPerfFile(name))
{
return new Check(this, name);
}
throw new KeyNotFoundException();
}
public override HttpResponse GetResponse(HttpRequest httpRequest, Queue<string> pathStack)
{
if (pathStack.Count == 0)
return GetResponse(httpRequest);
string perfName = WebUtility.UrlDecode(String.Join("/", pathStack));
Resource perfResource = GetResource(perfName);
return perfResource.GetResponse(httpRequest);
}
public override HttpResponse GetResponse(HttpRequest httpRequest)
{
HttpResponse response = new HttpResponse(httpRequest);
response.SetHeader("content-type", "application/json");
response.ContentWriter.Write(JSONMapper.DefaultMapper.ToJson(SkyScanner.Instance.Checker.PerfNames));
return response;
}
class Check : Resource
{
public Check(Checks container,String perfName)
:base(container,perfName)
{}
public override void AddResource(Resource resource) => throw new NotImplementedException();
public override bool Contains(string name) => throw new NotImplementedException();
public override IEnumerable<Resource> GetResources() => throw new NotImplementedException();
public override void RemoveResource(Resource resource) => throw new NotImplementedException();
public override HttpResponse GetResponse(HttpRequest httpRequest)
{
PerfFile perfFile = SkyScanner.Instance.Checker.GetPerfFile(Name);
PerfValue[] perfValues = null;
int interval = 3600;
if (httpRequest.Query.ContainsKey("interval"))
interval = int.Parse(httpRequest.Query["interval"]);
lock (perfFile)
{
perfFile.EnsureOpen();
perfValues = perfFile.QueryTime(interval, 0);
perfFile.Close();
}
HttpResponse response = new HttpResponse(httpRequest);
response.SetHeader("content-type", "application/json");
response.ContentWriter.Write(JSONMapper.DefaultMapper.ToJson(perfValues).ToString());
return response;
}
}
}
}
}

257
http/CrawlerApi.cs
View File

@ -1,145 +1,144 @@
using System; //using System;
using ln.http.resources; //using ln.http.resources;
using ln.skyscanner.entities; //using ln.skyscanner.entities;
using System.Linq; //using System.Linq;
using ln.skyscanner.crawl; //using ln.types;
using ln.types; //using System.Net;
using System.Net; //using ln.types.threads;
using ln.types.threads; //using ln.types.net;
using ln.types.net;
namespace ln.skyscanner.http //namespace ln.skyscanner.http
{ //{
public class CrawlerApi : JsonCallResource // public class CrawlerApi : JsonCallResource
{ // {
public SkyScanner SkyScanner { get; } // public SkyScanner SkyScanner { get; }
public Crawler Crawler => SkyScanner.Crawler; // public Crawler Crawler => SkyScanner.Crawler;
public CrawlerApi(Resource container, SkyScanner skyScanner) // public CrawlerApi(Resource container, SkyScanner skyScanner)
: base(container, "crawler") // : base(container, "crawler")
{ // {
SkyScanner = skyScanner; // SkyScanner = skyScanner;
} // }
[Callable] // [Callable]
public CrawledSubnet[] GetSubnets() // public CrawledSubnet[] GetSubnets()
{ // {
return SkyScanner.Instance.Entities.CrawledSubnets.ToArray(); // return SkyScanner.Instance.Entities.CrawledSubnets.ToArray();
} // }
[Callable] // [Callable]
public CrawledHost[] GetHosts() // public CrawledHost[] GetHosts()
{ // {
CrawledHost[] result = SkyScanner.Instance.Entities.CrawledHosts.ToArray(); // CrawledHost[] result = SkyScanner.Instance.Entities.CrawledHosts.ToArray();
return result; // return result;
} // }
[Callable] // [Callable]
public CrawledHost GetHostByIP(string _ip) // public CrawledHost GetHostByIP(string _ip)
{ // {
return SkyScanner.Crawler.FindHostForIP(IPv4.Parse(_ip)); // return SkyScanner.Crawler.FindHostForIP(IPv4.Parse(_ip));
} // }
[Callable] // [Callable]
public CrawledHost AddHost(string _ip, string name) // public CrawledHost AddHost(string _ip, string name)
{ // {
IPv4 ip = IPv4.Parse(_ip); // IPv4 ip = IPv4.Parse(_ip);
CrawledHost host = SkyScanner.Crawler.FindHostForIP(ip); // CrawledHost host = SkyScanner.Crawler.FindHostForIP(ip);
host.Name = name; // host.Name = name;
SkyScanner.Instance.Entities.CrawledHosts.Upsert(host); // SkyScanner.Instance.Entities.CrawledHosts.Upsert(host);
return host; // return host;
} // }
[Callable] // [Callable]
public void TriggerRecrawl() // public void TriggerRecrawl()
{ // {
foreach (CrawledHost host in SkyScanner.Instance.Entities.CrawledHosts.ToArray()) // foreach (CrawledHost host in SkyScanner.Instance.Entities.CrawledHosts.ToArray())
{ // {
host.NextCheck = DateTime.Now; // host.NextCheck = DateTime.Now;
SkyScanner.Instance.Entities.CrawledHosts.Upsert(host); // SkyScanner.Instance.Entities.CrawledHosts.Upsert(host);
} // }
} // }
[Callable] // [Callable]
public void Crawl(string _id) // public void Crawl(string _id)
{ // {
Guid hostID = Guid.Parse(_id); // Guid hostID = Guid.Parse(_id);
SkyScanner.Crawler.Crawl(hostID); // SkyScanner.Crawler.Crawl(hostID);
} // }
[Callable] // [Callable]
public PoolJob[] GetCurrentPoolJobs() // public PoolJob[] GetCurrentPoolJobs()
{ // {
return SkyScanner.Crawler.CurrentJobs; // return SkyScanner.Crawler.CurrentJobs;
} // }
[Callable] // [Callable]
public PoolJob[] GetQueuedJobs() // public PoolJob[] GetQueuedJobs()
{ // {
return SkyScanner.Crawler.QueuedJobs; // return SkyScanner.Crawler.QueuedJobs;
} // }
[Callable] // [Callable]
public bool GetCrawlSubnets() // public bool GetCrawlSubnets()
{ // {
return SkyScanner.Crawler.CrawlSubnets; // return SkyScanner.Crawler.CrawlSubnets;
} // }
[Callable] // [Callable]
public void SetCrawlSubnets(bool enable) // public void SetCrawlSubnets(bool enable)
{ // {
SkyScanner.Crawler.CrawlSubnets = enable; // SkyScanner.Crawler.CrawlSubnets = enable;
} // }
[Callable] // [Callable]
public bool GetCrawlHosts() // public bool GetCrawlHosts()
{ // {
return SkyScanner.Crawler.CrawlHosts; // return SkyScanner.Crawler.CrawlHosts;
} // }
[Callable] // [Callable]
public void SetCrawlHosts(bool enable) // public void SetCrawlHosts(bool enable)
{ // {
SkyScanner.Crawler.CrawlHosts = enable; // SkyScanner.Crawler.CrawlHosts = enable;
} // }
[Callable] // [Callable]
public void ToggleCrawlHosts() // public void ToggleCrawlHosts()
{ // {
SkyScanner.Crawler.CrawlHosts = !SkyScanner.Crawler.CrawlHosts; // SkyScanner.Crawler.CrawlHosts = !SkyScanner.Crawler.CrawlHosts;
} // }
[Callable] // [Callable]
public void ToggleCrawlSubnets() // public void ToggleCrawlSubnets()
{ // {
SkyScanner.Crawler.CrawlSubnets = !SkyScanner.Crawler.CrawlSubnets; // SkyScanner.Crawler.CrawlSubnets = !SkyScanner.Crawler.CrawlSubnets;
} // }
[Callable] // [Callable]
public Network4[] GetBlockedNetworks() // public Network4[] GetBlockedNetworks()
{ // {
return SkyScanner.Instance.Entities.BlockedNetworks.ToArray(); // return SkyScanner.Instance.Entities.BlockedNetworks.ToArray();
} // }
[Callable] // [Callable]
public void AddBlockedNetwork(string _network) // public void AddBlockedNetwork(string _network)
{ // {
Network4 blocked = Network4.Parse(_network); // Network4 blocked = Network4.Parse(_network);
if (!SkyScanner.Instance.Entities.BlockedNetworks.Contains(blocked)) // if (!SkyScanner.Instance.Entities.BlockedNetworks.Contains(blocked))
{ // {
SkyScanner.Instance.Entities.BlockedNetworks.Insert(blocked); // SkyScanner.Instance.Entities.BlockedNetworks.Insert(blocked);
} // }
} // }
[Callable] // [Callable]
public void RemoveBlockedNetwork(string _network) // public void RemoveBlockedNetwork(string _network)
{ // {
Network4 blocked = Network4.Parse(_network); // Network4 blocked = Network4.Parse(_network);
foreach (Network4 blockedNetwork in SkyScanner.Instance.Entities.BlockedNetworks) // foreach (Network4 blockedNetwork in SkyScanner.Instance.Entities.BlockedNetworks)
{ // {
if (blockedNetwork.Equals(blocked)) // if (blockedNetwork.Equals(blocked))
{ // {
SkyScanner.Instance.Entities.BlockedNetworks.Delete(blockedNetwork); // SkyScanner.Instance.Entities.BlockedNetworks.Delete(blockedNetwork);
return; // return;
} // }
} // }
} // }
} // }
} //}

52
http/NetworkApi.cs
View File

@ -17,42 +17,40 @@ namespace ln.skyscanner.http
{ {
public class NetworkApi : JsonCallResource public class NetworkApi : JsonCallResource
{ {
public GlobalNetwork GlobalNetwork => SkyScanner.Instance.Entities.GlobalNetwork;
public NetworkApi(SkyScannerHttpApi container) public NetworkApi(SkyScannerHttpApi container)
:base(container,"network") :base(container,"network")
{ {
} }
[Callable] //[Callable]
public Node[] GetNeighbors(string pid) //public Node[] GetNeighbors(string pid)
{ //{
Guid nodeID = Guid.Parse(pid); // Guid nodeID = Guid.Parse(pid);
return GlobalNetwork.FindNeighbors(GlobalNetwork.GetNode(nodeID)).ToArray(); // return GlobalNetwork.FindNeighbors(GlobalNetwork.GetNode(nodeID)).ToArray();
} //}
[Callable] //[Callable]
public Node[] GetHostsInSubnet(string _network) //public Node[] GetHostsInSubnet(string _network)
{ //{
Network4 network = Network4.Parse(_network); // Network4 network = Network4.Parse(_network);
return GlobalNetwork.FindHostsInSubnet(network).ToArray(); // return GlobalNetwork.FindHostsInSubnet(network).ToArray();
} //}
[Callable] //[Callable]
public HopMap.HopNode[] GetHopTable() //public HopMap.HopNode[] GetHopTable()
{ //{
HopMap hopMap = new HopMap(GlobalNetwork.FindNodeByIP(IPv4.Parse("10.10.10.2"))); // HopMap hopMap = new HopMap(GlobalNetwork.FindNodeByIP(IPv4.Parse("10.10.10.2")));
return hopMap.HopNodes; // return hopMap.HopNodes;
} //}
[Callable] //[Callable]
public HopMap.HopNode[] GetHopTable(string pid) //public HopMap.HopNode[] GetHopTable(string pid)
{ //{
Guid persistenceID = Guid.Parse(pid); // Guid persistenceID = Guid.Parse(pid);
HopMap hopMap = new HopMap(GlobalNetwork.GetNode(persistenceID)); // HopMap hopMap = new HopMap(GlobalNetwork.GetNode(persistenceID));
return hopMap.HopNodes; // return hopMap.HopNodes;
} //}
} }
} }

5
http/SkyScannerHttpApi.cs
View File

@ -13,11 +13,6 @@ namespace ln.skyscanner.http
:base(skyScannerHttpApplication.RootResource,"api") :base(skyScannerHttpApplication.RootResource,"api")
{ {
SkyScanner = skyScannerHttpApplication.SkyScanner; SkyScanner = skyScannerHttpApplication.SkyScanner;
new SkyScannerHttpManagement(this,SkyScanner);
new CrawlerApi(this, SkyScanner);
new NetworkApi(this);
new CheckerApi(this);
} }

5
http/SkyScannerHttpApplication.cs
View File

@ -1,14 +1,11 @@
using System; using System;
using ln.http;
using ln.http.resources; using ln.http.resources;
using System.IO; using System.IO;
using ln.skyscanner.entities; using ln.skyscanner.entities;
using ln.skyscanner.checks; using ln.skyscanner.checks;
using ln.skyscanner.crawl;
using ln.types.net; using ln.types.net;
using System.Linq; using System.Linq;
using System.Collections.Generic; using ln.skyscanner.perfvalue;
using ln.types.odb;
namespace ln.skyscanner.http namespace ln.skyscanner.http
{ {
public class SkyScannerHttpApplication : ResourceApplication public class SkyScannerHttpApplication : ResourceApplication

36
ln.skyscanner.csproj
View File

@ -62,8 +62,6 @@
<Compile Include="identify\NodeIdentifier.cs" /> <Compile Include="identify\NodeIdentifier.cs" />
<Compile Include="Target.cs" /> <Compile Include="Target.cs" />
<Compile Include="entities\Node.cs" /> <Compile Include="entities\Node.cs" />
<Compile Include="crawl\Crawler.cs" />
<Compile Include="crawl\CrawledHost.cs" />
<Compile Include="entities\Subnet.cs" /> <Compile Include="entities\Subnet.cs" />
<Compile Include="entities\NetworkInterface.cs" /> <Compile Include="entities\NetworkInterface.cs" />
<Compile Include="Check.cs" /> <Compile Include="Check.cs" />
@ -73,43 +71,29 @@
<Compile Include="http\SkyScannerHttpApi.cs" /> <Compile Include="http\SkyScannerHttpApi.cs" />
<Compile Include="http\SkyScannerHttpManagement.cs" /> <Compile Include="http\SkyScannerHttpManagement.cs" />
<Compile Include="http\CrawlerApi.cs" /> <Compile Include="http\CrawlerApi.cs" />
<Compile Include="crawl\CrawlNetwork.cs" />
<Compile Include="crawl\Crawl.cs" />
<Compile Include="crawl\service\SSH.cs" />
<Compile Include="crawl\service\SNMP.cs" />
<Compile Include="crawl\service\ICMP.cs" />
<Compile Include="crawl\service\RFC1213.cs" />
<Compile Include="entities\GlobalNetwork.cs" /> <Compile Include="entities\GlobalNetwork.cs" />
<Compile Include="crawl\CrawledSubnet.cs" />
<Compile Include="entities\HopMap.cs" /> <Compile Include="entities\HopMap.cs" />
<Compile Include="SkyEntities.cs" />
<Compile Include="http\NetworkApi.cs" /> <Compile Include="http\NetworkApi.cs" />
<Compile Include="crawl\service\CrawlService.cs" />
<Compile Include="crawl\service\TCP.cs" />
<Compile Include="entities\ConfiguredIP.cs" /> <Compile Include="entities\ConfiguredIP.cs" />
<Compile Include="crawl\service\HTTP.cs" />
<Compile Include="crawl\service\Ubiquity.cs" />
<Compile Include="checks\SkyChecker.cs" />
<Compile Include="checks\SkyCheck.cs" /> <Compile Include="checks\SkyCheck.cs" />
<Compile Include="checks\CheckJob.cs" /> <Compile Include="checks\CheckJob.cs" />
<Compile Include="http\CheckerApi.cs" />
<Compile Include="checks\Ubiquiti.cs" /> <Compile Include="checks\Ubiquiti.cs" />
<Compile Include="checks\SkyCheckState.cs" /> <Compile Include="checks\SkyCheckState.cs" />
<Compile Include="entities\PointOfPresence.cs" /> <Compile Include="entities\PointOfPresence.cs" />
<Compile Include="entities\CheckSeverity.cs" /> <Compile Include="entities\CheckSeverity.cs" />
<Compile Include="entities\DeviceType.cs" /> <Compile Include="entities\DeviceType.cs" />
<Compile Include="checks\APC.cs" /> <Compile Include="checks\APC.cs" />
<Compile Include="checks\PerformanceValue.cs" />
<Compile Include="import\skytron\SkytronImport.cs" /> <Compile Include="import\skytron\SkytronImport.cs" />
<Compile Include="entities\L2Segment.cs" /> <Compile Include="entities\L2Segment.cs" />
<Compile Include="checks\Mimosa.cs" /> <Compile Include="checks\Mimosa.cs" />
<Compile Include="checks\SikluCheck.cs" /> <Compile Include="checks\SikluCheck.cs" />
<Compile Include="devices\Siklu.cs" /> <Compile Include="devices\Siklu.cs" />
<Compile Include="crawl\CredentialsGenerator.cs" />
<Compile Include="http\WebSocketInterface.cs" /> <Compile Include="http\WebSocketInterface.cs" />
<Compile Include="services\CheckService.cs" /> <Compile Include="services\CheckService.cs" />
<Compile Include="services\CrawlService.cs" /> <Compile Include="services\CrawlService.cs" />
<Compile Include="services\EntityService.cs" /> <Compile Include="services\EntityService.cs" />
<Compile Include="services\PerformanceValueService.cs" />
<Compile Include="perfvalue\PerformanceValue.cs" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<None Include="packages.config" /> <None Include="packages.config" />
@ -241,6 +225,21 @@
<None Include="templates\static\checks\node.html"> <None Include="templates\static\checks\node.html">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None> </None>
<None Include="templates\static\dist\moment-timezone-with-data.js" />
<None Include="crawl\Crawler.cs" />
<None Include="crawl\CrawledHost.cs" />
<None Include="crawl\CrawlNetwork.cs" />
<None Include="crawl\Crawl.cs" />
<None Include="crawl\CrawledSubnet.cs" />
<None Include="crawl\CredentialsGenerator.cs" />
<None Include="crawl\service\ICMP.cs" />
<None Include="crawl\service\RFC1213.cs" />
<None Include="crawl\service\CrawlService.cs" />
<None Include="crawl\service\HTTP.cs" />
<None Include="crawl\service\SSH.cs" />
<None Include="crawl\service\SNMP.cs" />
<None Include="crawl\service\TCP.cs" />
<None Include="crawl\service\Ubiquity.cs" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<Folder Include="identify\" /> <Folder Include="identify\" />
@ -262,6 +261,7 @@
<Folder Include="import\skytron\" /> <Folder Include="import\skytron\" />
<Folder Include="devices\" /> <Folder Include="devices\" />
<Folder Include="services\" /> <Folder Include="services\" />
<Folder Include="perfvalue\" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\ln.snmp\ln.snmp.csproj"> <ProjectReference Include="..\ln.snmp\ln.snmp.csproj">

17
checks/PerformanceValue.cs → perfvalue/PerformanceValue.cs
View File

@ -9,7 +9,9 @@
// **/ // **/
using System; using System;
using ln.perfdb.storage; using ln.perfdb.storage;
namespace ln.skyscanner.checks using ln.skyscanner.checks;
namespace ln.skyscanner.perfvalue
{ {
public class PerformanceValue public class PerformanceValue
{ {
@ -39,19 +41,6 @@ namespace ln.skyscanner.checks
CritUpper = cUpper; CritUpper = cUpper;
} }
public void WritePerfValue(double value)
{
LastValue = value;
PerfFile perfFile = SkyScanner.Instance.Checker.GetPerfFile(PerfPath);
lock (perfFile)
{
perfFile.EnsureOpen();
perfFile.Write(DateTimeOffset.Now.ToUnixTimeSeconds(), value);
perfFile.Close();
}
}
public CheckState CheckState public CheckState CheckState
{ {
get get

72
services/CheckService.cs
View File

@ -8,48 +8,57 @@ using ln.skyscanner.checks;
using System.Collections.Generic; using System.Collections.Generic;
using ln.types.threads; using ln.types.threads;
using System.Linq; using System.Linq;
using ln.types.odb; using ln.snmp;
using ln.types.odb.ng;
namespace ln.skyscanner.services namespace ln.skyscanner.services
{ {
public class CheckService : ApplicationServiceBase public class CheckService : ApplicationServiceBase
{ {
public static bool DEBUG = false;
Pool checkPool; Pool checkPool;
EntityService entityService; EntityService entityService;
PerformanceValueService performanceValueService;
public EntityService EntityService => entityService;
public PerformanceValueService PerformanceValueService => performanceValueService;
public SNMPEngine SNMPEngine { get; private set; }
SkyScanner.Service coreService;
CheckServiceRPC RPC;
public CheckService() public CheckService()
:base("Check Service") :base("Check Service")
{ {
DependOnService<SkyScanner.Service>();
DependOnService<EntityService>(); DependOnService<EntityService>();
DependOnService<PerformanceValueService>();
} }
public override void ServiceMain(IApplicationInterface applicationInterface) public override void ServiceMain(IApplicationInterface applicationInterface)
{ {
coreService = Dependency<SkyScanner.Service>();
entityService = Dependency<EntityService>(); entityService = Dependency<EntityService>();
performanceValueService = Dependency<PerformanceValueService>();
Dictionary<Node, CheckJob> checkJobs = new Dictionary<Node, CheckJob>(); Dictionary<Node, CheckJob> checkJobs = new Dictionary<Node, CheckJob>();
HashSet<Node> currentNodes = new HashSet<Node>(); HashSet<Node> currentNodes = new HashSet<Node>();
checkPool = new Pool(64); checkPool = new Pool(96);
SNMPEngine = new SNMPEngine();
SNMPEngine.Timeout = 2000;
long nextMinute = DateTimeOffset.Now.ToUnixTimeMilliseconds(); long nextMinute = DateTimeOffset.Now.ToUnixTimeMilliseconds();
RPC = new CheckServiceRPC(this);
coreService.RPCContainer.Add("CheckService",RPC);
Ready(); Ready();
while (!StopRequested) while (!StopRequested)
{ {
/*
Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): scheduler save CheckStates");
lock (saveQueue)
{
foreach (SkyCheckState checkState in saveQueue)
{
SkyScanner.Instance.Entities.SkyCheckStates.Upsert(checkState);
}
saveQueue.Clear();
}
*/
Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): scheduler starts"); Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): scheduler starts");
currentNodes.Clear(); currentNodes.Clear();
@ -58,7 +67,7 @@ namespace ln.skyscanner.services
currentNodes.Add(node); currentNodes.Add(node);
if (!checkJobs.ContainsKey(node)) if (!checkJobs.ContainsKey(node))
{ {
CheckJob checkJob = new CheckJob(node); CheckJob checkJob = new CheckJob(this,node);
checkJobs.Add(node, checkJob); checkJobs.Add(node, checkJob);
} }
} }
@ -70,17 +79,20 @@ namespace ln.skyscanner.services
if (currentNodes.Count > 0) if (currentNodes.Count > 0)
{ {
foreach (Node node in currentNodes) foreach (Node node in currentNodes)
{
checkJobs.Remove(node); checkJobs.Remove(node);
}
} }
currentNodes.Clear(); currentNodes.Clear();
foreach (CheckJob checkJob in checkJobs.Values) foreach (CheckJob checkJob in checkJobs.Values)
{ {
checkJob.Prepare();
} }
Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): prepared {0} checks", checkJobs.Count); Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): currently knows {0} checks", checkJobs.Count);
Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): scheduled {0} checks", checkPool.Enqueue(checkJobs.Values)); Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): currently {0} checks alive", checkPool.NumQueuedJobs);
Logging.Log(LogLevel.INFO, "SkyChecker.scheduler(): scheduled {0} checks", checkPool.Enqueue(checkJobs.Values));
while (true) while (true)
@ -91,6 +103,7 @@ namespace ln.skyscanner.services
try try
{ {
Thread.Sleep((int)(nextMinute - DateTimeOffset.Now.ToUnixTimeMilliseconds())); Thread.Sleep((int)(nextMinute - DateTimeOffset.Now.ToUnixTimeMilliseconds()));
break;
} catch (ThreadInterruptedException) } catch (ThreadInterruptedException)
{ {
Logging.Log(LogLevel.INFO, "CheckService: scheduler was interrupted"); Logging.Log(LogLevel.INFO, "CheckService: scheduler was interrupted");
@ -100,13 +113,16 @@ namespace ln.skyscanner.services
} }
} }
coreService.RPCContainer.Remove("CheckService");
entityService = null; entityService = null;
performanceValueService = null;
coreService = null;
} }
public SkyCheckState[] GetCheckStates(Node node) public SkyCheckState[] GetCheckStates(Node node)
{ {
Query stateQuery = Query.Equals<SkyCheckState>("Node", node.ID); Query stateQuery = Query.Equals<SkyCheckState>("Node", node.ID);
SkyCheckState[] skyCheckStates = entityService.SkyCheckStates.Query(stateQuery).ToArray(); SkyCheckState[] skyCheckStates = entityService.SkyCheckStates.SelectQuery(stateQuery).ToArray();
Dictionary<string, SkyCheckState> checkStates = new Dictionary<string, SkyCheckState>(); Dictionary<string, SkyCheckState> checkStates = new Dictionary<string, SkyCheckState>();
foreach (SkyCheckState checkState in skyCheckStates) foreach (SkyCheckState checkState in skyCheckStates)
@ -135,6 +151,24 @@ namespace ln.skyscanner.services
this.checkService = checkService; this.checkService = checkService;
} }
public Node[] GetIssueList()
{
List<Node> nodes = new List<Node>();
foreach (Node node in checkService.EntityService.NodeCollection)
{
foreach (SkyCheckState checkState in node.CheckStates)
{
if (!String.Empty.Equals(node.Vendor) && ((checkState.CheckState != CheckState.OK) || ((checkState.UnchangedTime < TimeSpan.FromMinutes(10)) && (checkState.History.Length > 1)))){
nodes.Add(node);
break;
}
}
}
return nodes.ToArray();
}
} }
} }

128
services/EntityService.cs
View File

@ -3,43 +3,37 @@ using ln.application;
using ln.application.service; using ln.application.service;
using System.Threading; using System.Threading;
using System.IO; using System.IO;
using ln.types.odb; using ln.types.odb.ng;
using ln.types.odb.mapped;
using ln.skyscanner.entities; using ln.skyscanner.entities;
using ln.skyscanner.crawl;
using ln.types.net;
using ln.skyscanner.checks;
using ln.logging; using ln.logging;
using System.Collections.Generic;
using ln.types.odb.ng.storage;
namespace ln.skyscanner.services namespace ln.skyscanner.services
{ {
public class EntityService : ApplicationServiceBase public class EntityService : ApplicationServiceBase
{ {
public string BasePath { get; private set; } public string BasePath { get; private set; }
public ODB ODB { get; private set; } public SkyScanner.Service CoreService { get; private set; }
public RPC RPCInstance { get; private set; }
public ODBCollection<Node> NodeCollection { get; private set; }
public ODBCollection<Subnet> SubnetCollection { get; private set; }
public ODBCollection<PointOfPresence> PointOfPresenceCollection { get; private set; }
public ODBCollection<L2Segment> L2SegmentCollection { get; private set; }
public ODBCollection<CrawledHost> CrawledHosts { get; private set; }
public ODBCollection<CrawledSubnet> CrawledSubnets { get; private set; }
public ODBCollection<Network4> BlockedNetworks { get; private set; }
public ODBCollection<SkyCheckState> SkyCheckStates { get; private set; }
public IStorageContainer StorageContainer { get; private set; }
public IStorageContainer StorageSession { get; private set; }
public Mapper SessionMapper { get; private set; }
Queue<object> upsertQueue = new Queue<object>();
public EntityService() public EntityService()
:base("Entity Service") :base("Entity Service")
{ {
DependOnService<SkyScanner.Service>();
RPCInstance = new RPC(this);
} }
public override void ServiceMain(IApplicationInterface applicationInterface) public override void ServiceMain(IApplicationInterface applicationInterface)
{ {
CoreService = Dependency<SkyScanner.Service>();
BasePath = Path.Combine( BasePath = Path.Combine(
CurrentApplicationInterface.Arguments["base-path"].Value, CurrentApplicationInterface.Arguments["base-path"].Value,
"entities" "entities"
@ -47,48 +41,82 @@ namespace ln.skyscanner.services
Logging.Log(LogLevel.INFO, "Entity Service: Initializing"); Logging.Log(LogLevel.INFO, "Entity Service: Initializing");
ODB = new ODB(BasePath); StorageContainer = new FSStorageContainer(System.IO.Path.Combine(BasePath, "storage"));
StorageSession = new Session(StorageContainer);
SessionMapper = new Mapper(StorageSession);
NodeCollection = ODB.GetCollection<Node>(); StorageContainer.Open();
NodeCollection.EnableStrongCache(true); StorageSession.Open();
NodeCollection.EnsureIndex("uniqueIdentity", true);
NodeCollection.EnsureIndeces(
"PrimaryIP",
"Interfaces[].ConfiguredIPs[].IP",
"Interfaces[].ConfiguredIPs[].Network"
);
SubnetCollection = ODB.GetCollection<Subnet>();
SubnetCollection.EnsureIndeces("Network");
PointOfPresenceCollection = ODB.GetCollection<PointOfPresence>();
PointOfPresenceCollection.EnsureIndeces("ForeignName");
L2SegmentCollection = ODB.GetCollection<L2Segment>();
L2SegmentCollection.EnsureIndeces("Network", "PoPs");
CrawledHosts = ODB.GetCollection<CrawledHost>();
CrawledHosts.EnsureIndeces("PrimaryIP", "IPAddresses[]");
CrawledSubnets = ODB.GetCollection<CrawledSubnet>();
BlockedNetworks = ODB.GetCollection<Network4>("blockedNetworks");
SkyCheckStates = ODB.GetCollection<SkyCheckState>();
SkyCheckStates.EnableStrongCache(true);
SkyCheckStates.EnsureIndeces("Node");
SkyCheckStates.EnsureUniqueness("Node", "CheckName");
CoreService.RPCContainer.Add("entities",RPCInstance);
Ready(); Ready();
lock (Thread.CurrentThread) while (!StopRequested)
{ {
Monitor.Wait(Thread.CurrentThread); //object o;
//while (
// (o = DequeueUpsert()) != null
// )
//{
// ODB.GetCollection(o.GetType()).Upsert(o);
//}
lock (upsertQueue)
{
if (upsertQueue.Count == 0)
Monitor.Wait(upsertQueue,2500);
}
} }
CoreService.RPCContainer.Remove(RPCInstance);
CoreService = null;
SessionMapper = null;
StorageSession.Dispose();
StorageContainer.Dispose();
BasePath = null; BasePath = null;
} }
public void EnqueueUpsert<T>(T o) where T: class
{
//ObjectCollection<T> col = ODBMapper.GetCollection<T>();
//lock (upsertQueue)
//{
// upsertQueue.Enqueue(o);
// Monitor.Pulse(upsertQueue);
//}
}
public object DequeueUpsert()
{
lock (upsertQueue)
{
if (upsertQueue.Count == 0)
return null;
return upsertQueue.Dequeue();
}
}
public class RPC
{
EntityService EntityService { get; }
public RPC(EntityService entityService)
{
EntityService = entityService;
}
public Node GetNode(Guid nodeID)
{
return EntityService.SessionMapper.Load<Node>(nodeID);
}
}
} }
} }

117
services/PerformanceValueService.cs 100644
View File

@ -0,0 +1,117 @@
using System;
using ln.application;
using ln.application.service;
using System.Threading;
using ln.skyscanner.perfvalue;
using ln.perfdb.storage;
using System.Collections.Generic;
using System.Linq;
using System.IO;
namespace ln.skyscanner.services
{
public class PerformanceValueService : ApplicationServiceBase
{
public String BasePath { get; private set; }
private Dictionary<string, PerfFile> perfFiles = new Dictionary<string, PerfFile>();
public PerfFile[] PerfFiles => perfFiles.Values.ToArray();
SkyScanner.Service coreService;
public RPC RPCInstance { get; private set; }
public PerformanceValueService()
:base("PerformanceValueService")
{
DependOnService<SkyScanner.Service>();
RPCInstance = new RPC(this);
}
public override void ServiceMain(IApplicationInterface applicationInterface)
{
coreService = Dependency<SkyScanner.Service>();
BasePath = Path.Combine(
CurrentApplicationInterface.Arguments["base-path"].Value,
"perfdb"
);
coreService.RPCContainer.Add("perfValues", RPCInstance);
Ready();
while (!StopRequested)
{
lock (this)
{
Monitor.Wait(this);
}
}
coreService.RPCContainer.Remove(RPCInstance);
coreService = null;
}
public PerfFile GetPerfFile(params string[] perfPath)
{
lock (this)
{
string perfName = string.Join("/", perfPath);
if (!perfFiles.ContainsKey(perfName))
{
String perfDirectory = Path.Combine(BasePath, Path.Combine(perfPath.Take(perfPath.Length - 1).ToArray()));
if (!Directory.Exists(perfDirectory))
Directory.CreateDirectory(perfDirectory);
String perfFileName = Path.Combine(perfDirectory, String.Format("{0}.perf", perfPath[perfPath.Length - 1]));
PerfFile perfFile = new PerfFile(perfFileName);
perfFile.Open();
if (perfFile.FirstSection == null)
{
PerfFile.PerfFileSection s1 = new PerfFile.PerfFileSection(perfFile, null, TimeSpan.FromDays(28), 60, AggregationMethod.AVERAGE);
PerfFile.PerfFileSection s2 = new PerfFile.PerfFileSection(perfFile, s1, TimeSpan.FromDays(56), 300, AggregationMethod.AVERAGE);
PerfFile.PerfFileSection s3 = new PerfFile.PerfFileSection(perfFile, s2, TimeSpan.FromDays(84), 900, AggregationMethod.AVERAGE);
PerfFile.PerfFileSection s4 = new PerfFile.PerfFileSection(perfFile, s3, TimeSpan.FromDays(168), 1800, AggregationMethod.AVERAGE);
PerfFile.PerfFileSection s5 = new PerfFile.PerfFileSection(perfFile, s4, TimeSpan.FromDays(750), 3600, AggregationMethod.AVERAGE);
}
perfFile.Close();
perfFiles.Add(perfName, perfFile);
}
return perfFiles[perfName];
}
}
public void WriteValue(PerformanceValue performanceValue,double value)
{
performanceValue.LastValue = value;
PerfFile perfFile = GetPerfFile(performanceValue.PerfPath);
lock (perfFile)
{
perfFile.EnsureOpen();
perfFile.Write(DateTimeOffset.Now.ToUnixTimeSeconds(), value);
perfFile.Close();
}
}
public class RPC
{
PerformanceValueService PerformanceValueService { get; }
public RPC(PerformanceValueService performanceValueService)
{
PerformanceValueService = performanceValueService;
}
public PerfValue[] GetPerfData(string[] perfPath,int timeWindow = 3600)
{
PerfFile perfFile = PerformanceValueService.GetPerfFile(perfPath);
perfFile.EnsureOpen();
PerfValue[] perfData = perfFile.QueryTime(timeWindow, 0);
return perfData;
}
}
}
}

324
templates/static/checks/issues2.html
View File

@ -23,10 +23,15 @@
</table> </table>
<script type="text/javascript"> <script type="text/javascript">
var STATES = [ "OK", "WARN", "CRITICAL" ];
moment.defaultFormat = "DD.MM.YYYY HH:mm:ss"; moment.defaultFormat = "DD.MM.YYYY HH:mm:ss";
moment.tz.setDefault("Europe/Berlin");
var currentUpdateCycle = 0; var currentUpdateCycle = 0;
var issues = {} var listedNodes = {}
var charts = [] var charts = []
var chartTimeout = null; var chartTimeout = null;
@ -58,155 +63,116 @@
return r; return r;
} }
function graphLoaded(performanceValue, perfData)
function updateNode(node)
{ {
var chartColor = performanceValue.CheckState == "CRITICAL" ? '#FF0000' : (performanceValue.CheckState == "WARN") ? '#C0C000' : '#000000'; //console.log(JSON.stringify(node));
var perfID = encodeID(performanceValue.PerfName);
var htmlChart = $("#" + perfID );
if (!htmlChart.length)
htmlChart = $("#ChartBoxTemplate").clone().attr("id", perfID ).appendTo( $("#Charts") );
var chart = new Chart( $("#chart", htmlChart) , {
type: 'bar',
data: {
datasets: [
{
label: "-",
data: [],
backgroundColor: chartColor,
}
]
},
options: {
scales: {
yAxes: [
{
ticks: {
callback: ScaleSI,
beginAtZero: true,
}
}
],
xAxes: [{
type: 'time',
time: {
unit: "minute",
tooltipFormat: "DD.MM.YYYY HH:mm",
displayFormats: {
minute: "DD.MM.YYYY HH:mm"
},
parser: moment.unix
}
}]
},
responsive: true,
maintainAspectRatio: false
}
} );
chart.data.labels.length = 0;
chart.data.datasets[0].data.length = 0;
chart.data.datasets[0].label = performanceValue.PerfName;
$("#chart", htmlChart).data("chart", chart);
$.each( perfData, function(){ var IID = node.ID;
if (this.TimeStamp != 0) var currentNode = null;
chart.data.datasets[0].data.push( { x: this.TimeStamp, y: this.Value } );
});
chart.update();
}
function showCharts(issue)
{
var htmlCharts = $("#Charts");
$("canvas", htmlCharts).each(function(i,e){
var htmlCanvas = $(e);
htmlCanvas.data("chart").destroy();
htmlCanvas.data("chart",null);
});
htmlCharts.empty();
$.each( issue.PerformanceValues, function(){
var currentIssue = this;
skyapi().getJson("api/checker/checks/" + encodeURIComponent(this.PerfName) + "?interval=" + $("#interval").children("option:selected").val(), function(perfData){
graphLoaded(currentIssue, perfData);
});
});
}
function updateIssue(issue)
{
var IID = issue.ID;
var currentIssue = null;
if (issues[IID]) if (issues[IID])
{ {
currentIssue = issues[IID]; currentNode = listedNodes[IID];
} else { } else {
currentIssue = {}; currentNode = {};
issues[IID] = currentIssue; listedNodes[IID] = currentNode;
} }
currentIssue.issue = issue; currentNode.Node = node;
currentIssue.updateCyle = currentUpdateCycle; currentNode.updateCyle = currentUpdateCycle;
if (!currentIssue.html) if (!currentNode.html)
{ {
currentIssue.html = $("<tr></tr>"); currentNode.html = $("<tr></tr>");
for (var i=0;i<6;i++){ for (var i=0;i<6;i++){
$("<td></td>") $("<td></td>")
.appendTo(currentIssue.html); .appendTo(currentNode.html);
} }
currentIssue.html.data("issue", currentIssue); currentNode.html.data("node", currentNode);
currentIssue.html.appendTo($("#issues > tbody")); currentNode.html.appendTo($("#issues > tbody"));
} }
currentIssue.html.removeClass("issue-WARN issue-CRITICAL"); $($("td", currentNode.html).get(0))
currentIssue.html.addClass("issue-" + issue.CheckState );
var now = moment();
var m1 = issue.History.length > 0 ? moment(issue.History.slice(-1)[0].Timestamp) : moment();
var m2 = issue.History.length > 1 ? moment(issue.History.slice(-2)[0].Timestamp) : moment();
$($("td", currentIssue.html).get(0))
.empty() .empty()
.append( .append(
$("<a></a>") $("<a></a>")
.text(currentIssue.issue.Node.Name + " [ "+ currentIssue.issue.Node.UniqueIdentity +" ]") .text(node.Name + " [ "+ node.UniqueIdentity +" ]")
.attr("href","/static/checks/node.html#" + currentIssue.issue.Node.ID) .attr("href","/static/checks/node.html#" + node.ID)
.attr("target","_blank") .attr("target","_blank")
); );
$($("td", currentIssue.html).get(1))
.text(currentIssue.issue.Node.Comment);
$($("td", currentIssue.html).get(2))
.text(currentIssue.issue.CheckName);
$($("td", currentIssue.html).get(3))
.text( timespan((now - m1)/1000) + (issue.History.length > 1 ? " ( " + issue.History.slice(-2)[0].NewState + "=" + timespan((m1 - m2)/1000) + " )" : "") );
$($("td", currentIssue.html).get(4)).empty(); $($("td", currentNode.html).get(1))
$.each( currentIssue.issue.PerformanceValues, function(){ .text(node.Comment);
if (this.CheckState != "OK")
{
$($("td", currentIssue.html).get(4))
.append(
this.PerfName + "<br>"
);
}
});
$($("td", currentIssue.html).get(5)) $($("td", currentNode.html).get(2))
.empty();
$($("td", currentNode.html).get(3))
.empty();
$($("td", currentNode.html).get(4))
.empty(); .empty();
currentIssue.issue.Node.URIs.forEach(function(e,i){ var tzOffset = new Date().getTimezoneOffset() * 60;
currentNode.highestStateTime = 0;
currentNode.checkState = "OK";
$.each(node.CheckStates, function(){
var now = moment.unix((Date.now()/1000) - tzOffset);
var m1 = this.History.length > 0 ? moment.unix(this.History.slice(-1)[0].Timestamp) : moment();
var m2 = this.History.length > 1 ? moment.unix(this.History.slice(-2)[0].Timestamp) : moment();
var currentStateTime = (now - m1)/1000;
if (STATES.indexOf(currentNode.checkState) < STATES.indexOf(this.CheckState))
{
currentNode.checkState = this.CheckState;
currentNode.highestStateTime = 0;
}
if (currentNode.highestStateTime < currentStateTime)
currentNode.highestStateTime = currentStateTime;
$($("td", currentNode.html).get(2))
.append(
$("<span></span><br>")
.text(this.CheckName)
.addClass("issue-" + this.CheckState )
);
$($("td", currentNode.html).get(3))
.append(
$("<span></span><br>")
.text( timespan(currentStateTime) + (this.History.length > 1 ? " ( " + this.History.slice(-2)[0].NewState + "=" + timespan((m1 - m2)/1000) + " )" : "") )
.addClass("issue-" + this.CheckState )
);
$.each( this.PerformanceValues, function(){
if (this.CheckState != "OK")
{
$($("td", currentNode.html).get(4))
.append(
$("<span></span><br>")
.text(this.PerfName)
.addClass("issue-" + this.CheckState )
);
}
});
});
currentNode.html.removeClass("issue-WARN issue-CRITICAL issue-OK");
currentNode.html.addClass("issue-" + currentNode.checkState );
$($("td", currentNode.html).get(5))
.empty()
.text(currentNode.highestStateTime);
/*
currentNode.issue.Node.URIs.forEach(function(e,i){
if (e.Scheme == "http"){ if (e.Scheme == "http"){
var url = "http://" + e.Host + ":" + e.Port; var url = "http://" + e.Host + ":" + e.Port;
$($("td", currentIssue.html).get(5)) $($("td", currentNode.html).get(5))
.append( .append(
$("<a target=_blank></a>") $("<a target=_blank></a>")
.attr("href",url) .attr("href",url)
@ -214,83 +180,28 @@
); );
} }
}, this); }, this);
*/
/* $("h2", ne).text( issue.Node.DeviceType + " " + issue.CheckName + ": " + issue.Node.Name + " ( " + issue.Node.UniqueIdentity + " )").on("click", function(e){
showCharts($(this).parent().data("MyIssue"));
});
$("#LastCheckTime", ne).text(issue.LastCheckTime);
$("#UnchangedTime", ne).text(issue.UnchangedTime);
$("#Comment", ne).text(issue.Node.Comment);
var awebui = $("#webui", ne);
awebui.hide();
issue.Node.URIs.forEach(function(e,i){
if (e.Scheme == "http"){
awebui.show();
awebui.attr("href", "http://" + e.Host + ":" + e.Port);
}
}, this);
var histParent = $("#History", ne);
histParent.empty();
var lastHistTime = moment();
$.each( issue.History.slice().reverse(), function(){
var m = moment(this.Timestamp);
var d = lastHistTime == null ? "" : lastHistTime.diff(m, 'minutes');
lastHistTime = m;
$("<span class='" + this.NewState + "'>" + m.format() + " [ " + d + "min ]</span><br>").appendTo( histParent );
});
var pvparent = $("#PerfValues", ne);
pvparent.empty();
$.each( issue.PerformanceValues, function(){
pvparent.append( $("<span class='" + this.CheckState + "'>" + this.PerfName + "</span><br>") );
});
*/
} }
function loadNextChart() function updateIssueList(nodes)
{
if (charts.length == 0)
{
clearTimeout(chartTimeout);
chartTimeout = null;
} else {
if (chartTimeout == null)
{
chartTimeout = setTimeout( loadNextChart, 100 );
}
next = charts.shift();
skyapi().getJson("api/checker/checks/" + encodeURIComponent(next.performanceValue.PerfName) + "?interval=" + $("#interval").children("option:selected").val(), function(perfData){
graphLoaded(next.pv, next.performanceValue, perfData);
});
}
}
function updateIssueList(issueList)
{ {
var startTime = new Date().getTime();
currentUpdateCycle++; currentUpdateCycle++;
$.each( issueList , function(){ //console.log(JSON.stringify(nodes));
// if (this.Node.DeviceType != "UNKNOWN")
updateIssue(this); $.each( nodes , function(){
updateNode(this);
}); });
$.each( $("#issues > tbody > tr"), function(){ $.each( $("#issues > tbody > tr"), function(){
var currentIssue = $(this).data("issue"); var currentNode = $(this).data("node");
if (currentIssue.updateCyle != currentUpdateCycle) if (currentNode.updateCyle != currentUpdateCycle)
{ {
delete issues[currentIssue.issue.ID]; delete issues[currentNode.Node.ID];
currentIssue.html.detach(); currentNode.html.detach();
} }
}); });
@ -298,12 +209,12 @@
var sortList = Array.prototype.sort.bind(rows); var sortList = Array.prototype.sort.bind(rows);
sortList(function(a,b){ sortList(function(a,b){
var aissue = $(a).data("issue"); var anode = $(a).data("node");
var bissue = $(b).data("issue"); var bnode = $(b).data("node");
var ta = aissue.issue.History.length > 0 ? moment(aissue.issue.History.slice(-1)[0].Timestamp) : moment();
var tb = bissue.issue.History.length > 0 ? moment(bissue.issue.History.slice(-1)[0].Timestamp) : moment();
var ta = anode.highestStateTime;
var tb = bnode.highestStateTime;
if (ta < tb) if (ta < tb)
return -1; return -1;
else if (ta > tb) else if (ta > tb)
@ -317,15 +228,28 @@
$("#nWARN").text( $("#issues > tbody > .issue-WARN").length ); $("#nWARN").text( $("#issues > tbody > .issue-WARN").length );
$("#nOK").text( $("#issues > tbody > .issue-OK").length ); $("#nOK").text( $("#issues > tbody > .issue-OK").length );
$("#lastUpdate").text( moment().format() ); $("#lastUpdate").text( moment().format() );
var stopTime = new Date().getTime();
console.log("Refresh Time: " + (stopTime - startTime) + "ms");
} }
skyapi().getJson("/checker/Issues", updateIssueList );
function refresh() function refresh()
{ {
skyapi().getJson("/checker/Issues", updateIssueList ); LN().rpc("CheckService","GetIssueList",[],function(r,e){
if (e){
alert("Error fetching current issue list!\n" + JSON.stringify(e));
} else {
updateIssueList(r);
}
});
setTimeout(refresh, 15000);
} }
skyapi().addRefresh( refresh, 15 ); refresh();
// skyapi().addRefresh( refresh, 15 );
</script> </script>

123
templates/static/checks/node.html
View File

@ -1,6 +1,23 @@
<%frame "frame.html"%> <%frame "frame.html"%>
<h1>Knoten <span id="nodeName"></span> (<span id="lastUpdate"></span>)</h1> <h1>Knoten <span id="nodeName"></span> (<span id="comment"></span>)</h1>
<div class="flex row wrap">
<div class="grow">
Current State: <span id="currentState"></span><br>
Sender: <span id="sender"></span><br>
</div>
<div class="grow">
<a href="" class="protolink">
<div>HTTP:</div>
</a>
<a href="" class="protolink">
<div>SSH:</div>
</a>
<a href="" class="protolink">
<div>TELNET:</div>
</a>
</div>
</div>
<div> <div>
Graphen Intervall: Graphen Intervall:
@ -14,6 +31,8 @@
</div> </div>
<div class="flex row wrap"> <div class="flex row wrap">
<div> <div>
<span class="h1">Checks</span>
<table id="checkStates"> <table id="checkStates">
<thead> <thead>
<tr> <tr>
@ -33,17 +52,86 @@
<script type="text/javascript"> <script type="text/javascript">
moment.defaultFormat = "DD.MM.YYYY HH:mm:ss"; moment.defaultFormat = "DD.MM.YYYY HH:mm:ss";
var myNodeID = window.location.hash.slice(1); var myNodeID = window.location.hash.slice(1);
skyapi().getJson("/collections/Node/" + myNodeID, function(node){ function loadNode(nodeID){
$("#nodeName").text(node.Name); LN().rpc("entities","GetNode",[myNodeID,],function(node,e){
$("#lastUpdate").text(node.LastUpdate); if (e){
console.log("Error loading Node " + myNodeID);
console.log(JSON.stringify(e));
} else {
var tzOffset = new Date().getTimezoneOffset() * 60;
var now = moment.unix((Date.now()/1000) - tzOffset);
console.log(JSON.stringify(node));
});
$("#nodeName").text(node.Name);
$("#comment").text(node.comment);
$("#currentState").text(node.CheckState);
$("#checkStates > tbody").empty();
$.each( node.CheckStates, function(){
var row = $("<tr></tr>")
.append(
$("<td></td>")
.addClass("issue-" + this.CheckState)
.text(this.CheckName)
)
.append(
$("<td></td>")
.addClass("issue-" + this.CheckState)
.text(timespan(this.UnchangedTime))
)
.appendTo($("#checkStates > tbody"));
var tdHist = $("<td></td>").appendTo(row);
var lastTimestamp = now;
Array.reverse(this.History);
$.each( this.History, function(){
var currentTimestamp = moment.unix(this.Timestamp);
var age = (lastTimestamp - currentTimestamp) / 1000;
tdHist.append(
$("<span></span><br>")
.addClass("issue-" + this.NewState )
.text(this.NewState + "[ " + timespan(age) + " ]")
);
lastTimestamp = currentTimestamp;
});
var tdPerfValues = $("<td></td>").appendTo(row);
$.each( this.PerformanceValues, function(){
var pv = this;
tdPerfValues.append(
$("<span></span><br>")
.text(this.PerfName)
.addClass("issue-" + this.CheckState)
.click(function(ev){
loadGraph(pv);
ev.preventDefault();
})
);
});
});
}
});
}
loadNode(myNodeID);
/* setInterval( function(){
loadNode(myNodeID);
}, 1000 );
*/
/*
skyapi().getJson("/checker/CheckStatesByHostID?nodeID=" + myNodeID,function(checkStates){ skyapi().getJson("/checker/CheckStatesByHostID?nodeID=" + myNodeID,function(checkStates){
$("#checkStates > tbody").empty(); $("#checkStates > tbody").empty();
@ -97,12 +185,7 @@
},this); },this);
}); });
*/
var currentUpdateCycle = 0;
var issues = {}
var charts = []
var chartTimeout = null;
function ntos(n,l){ function ntos(n,l){
n = n.toString(); n = n.toString();
while (n.length < l){ while (n.length < l){
@ -110,7 +193,6 @@
} }
return n; return n;
} }
function timespan(value){ function timespan(value){
var days, hours, minutes, seconds; var days, hours, minutes, seconds;
value = parseInt(value); value = parseInt(value);
@ -133,6 +215,8 @@
function graphLoaded(graphDiv, performanceValue, perfData) function graphLoaded(graphDiv, performanceValue, perfData)
{ {
console.log("graph " + performanceValue.PerfName + " has " + perfData.length + "records.");
var chartColor = performanceValue.CheckState == "CRITICAL" ? '#FF0000' : (performanceValue.CheckState == "WARN") ? '#C0C000' : '#000000'; var chartColor = performanceValue.CheckState == "CRITICAL" ? '#FF0000' : (performanceValue.CheckState == "WARN") ? '#C0C000' : '#000000';
var htmlChart = $("canvas", graphDiv); var htmlChart = $("canvas", graphDiv);
var chart = htmlChart.data("chart"); var chart = htmlChart.data("chart");
@ -204,10 +288,13 @@
.append("<canvas></canvas"); .append("<canvas></canvas");
} }
skyapi().getJson("api/checker/checks/" + encodeURIComponent(graphName) + "?interval=" + $("#interval").children("option:selected").val(), function(perfData){ LN().rpc("perfValues","GetPerfData",[performanceValue.PerfPath,$("#interval").children("option:selected").val()],function(perfData,e){
if (e){
console.log(e);
} else {
graphLoaded(graphDiv, performanceValue, perfData); graphLoaded(graphDiv, performanceValue, perfData);
}); }
});
} }
function updateGraphs(){ function updateGraphs(){

37
templates/static/css/style.css
View File

@ -45,6 +45,15 @@ h2 {
font-size: 14px; font-size: 14px;
} }
.h1 {
display: block;
width: 100%;
border-bottom: 2px solid black;
margin-bottom: 4px;
font-size: 18px;
}
a { a {
text-decoration: none; text-decoration: none;
color: inherit; color: inherit;
@ -496,3 +505,31 @@ span#nWARN, span#nCRITICAL, span#nOK {
border: 1px solid white; border: 1px solid white;
background-color: red; background-color: red;
} }
.protolink {
position: relative;
display: table-cell;
width: 64px;
height: 64px;
border: 3px solid white;
border-radius: 28px;
text-align: center;
vertical-align: middle;
background-color: #009ee3;
color: white;
}
.protolink > div {
font-family: 'Courier New', Courier, monospace;
font-size: 14px;
font-weight: bold;
font-style: normal;
text-decoration: underline;
}

1224
templates/static/dist/moment-timezone-with-data.js vendored 100644
View File

File diff suppressed because one or more lines are too long

1
templates/static/frame.html
View File

@ -16,6 +16,7 @@
<script type="text/javascript" src="/static/dist/jquery.min.js"></script> <script type="text/javascript" src="/static/dist/jquery.min.js"></script>
<script type="text/javascript" src="/static/dist/jquery-ui.min.js"></script> <script type="text/javascript" src="/static/dist/jquery-ui.min.js"></script>
<script type="text/javascript" src="/static/dist/moment-with-locales.js"></script> <script type="text/javascript" src="/static/dist/moment-with-locales.js"></script>
<script type="text/javascript" src="/static/dist/moment-timezone-with-data.js"></script>
<script type="text/javascript" src="/static/dist/datatables.min.js"></script> <script type="text/javascript" src="/static/dist/datatables.min.js"></script>
<script type="text/javascript" src="/static/dist/percentageBars.js"></script> <script type="text/javascript" src="/static/dist/percentageBars.js"></script>
<script type="text/javascript" src="/static/dist/Chart.min.js"></script> <script type="text/javascript" src="/static/dist/Chart.min.js"></script>

16
templates/static/system/index.html
View File

@ -47,23 +47,23 @@
// $("#cbCrawlHosts").on("click", function(){ skyapi().call("api/crawler","ToggleCrawlHosts"); } ); // $("#cbCrawlHosts").on("click", function(){ skyapi().call("api/crawler","ToggleCrawlHosts"); } );
// $("#cbCrawlSubnets").on("click", function(){ skyapi().call("api/crawler","ToggleCrawlSubnets"); } ); // $("#cbCrawlSubnets").on("click", function(){ skyapi().call("api/crawler","ToggleCrawlSubnets"); } );
function showServices(json){ function showServices(result,error){
if (json.result) if (result)
{ {
$("#ServiceTable > tbody").empty(); $("#ServiceTable > tbody").empty();
json.result.forEach(element => { result.forEach(element => {
var row = $("<tr></tr>") var row = $("<tr></tr>")
.attr("id",$.escapeSelector(element)) .attr("id",$.escapeSelector(element))
.append($("<td></td>").text(element)); .append($("<td></td>").text(element));
$("#ServiceTable > tbody").append(row); $("#ServiceTable > tbody").append(row);
LN().rpc("ServiceContainer","GetServiceDefinition",[element,],function(json){ LN().rpc("ServiceContainer","GetServiceDefinition",[element,],function(result,error){
console.log(JSON.stringify(json)); console.log(JSON.stringify(result));
row.append( row.append(
$("<td></td>").attr("bool",json.result.IsLoaded), $("<td></td>").attr("bool",result.IsLoaded),
$("<td></td>").attr("bool",json.result.IsAlive), $("<td></td>").attr("bool",result.IsAlive),
$("<td></td>").attr("bool",json.result.IsReady) $("<td></td>").attr("bool",result.IsReady)
); );
}); });