Skytron
/
ln.skyscanner
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

WIP

broken
Harald Wolff 2019-03-15 15:35:44 +01:00
parent e991a74fa8
commit 88c33ee53b
12 changed files with 15238 additions and 301 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
SkyScanner.cs
View File

@ -9,7 +9,7 @@ using ln.types.threads;
using System.Threading;
namespace ln.skyscanner
{
public enum ComponentState { STOPPED, INITIALIZED, STARTED, FAILED }
public enum ComponentState { STOPPED, INITIALIZED, STARTED, FAILED, STOPPING }
public class SkyScanner
{
@ -50,7 +50,7 @@ namespace ln.skyscanner
StopCrawler();
StopHttpServer();
new Thread(() => ConveniencePool.NumThreads = 0).Start();
new Thread(() => ConveniencePool.Close()).Start();
}
@ -101,11 +101,7 @@ namespace ln.skyscanner
{
if (Crawler != null)
{
if (Crawler.ThreadStates.Length > 0)
{
return ComponentState.STARTED;
}
return ComponentState.INITIALIZED;
return Crawler.CrawlerState;
}
return ComponentState.STOPPED;
}

60
crawl/Crawler.cs
View File

@ -38,38 +38,53 @@ namespace ln.skyscanner.crawl
bool stopping;
Pool crawlThreadPool = new Pool();
Pool crawlThreadPool = new Pool(12);
public PoolThreadState[] ThreadStates => crawlThreadPool.ThreadStates;
public int QueuedJobs => crawlThreadPool.QueuedJobs;
public PoolJob[] CurrentJobs => crawlThreadPool.CurrentPoolJobs;
public PoolJob[] QueuedJobs => crawlThreadPool.QueuedJobs;
public DiskObject<CrawlPool> _CrawlPool;
public CrawlPool CrawlPool => _CrawlPool.Instance;
public SNMPEngine SNMPEngine { get; }
public SNMPEngine SNMPEngine { get; private set; }
Thread threadScheduler;
public Crawler(SkyScanner skyScanner)
{
SkyScanner = skyScanner;
try
{
BasePath = Path.Combine(skyScanner.BasePath, "crawler");
BasePath = Path.Combine(skyScanner.BasePath, "crawler");
if (!Directory.Exists(BasePath))
Directory.CreateDirectory(BasePath);
if (!Directory.Exists(PerfPath))
Directory.CreateDirectory(PerfPath);
if (!Directory.Exists(BasePath))
Directory.CreateDirectory(BasePath);
if (!Directory.Exists(PerfPath))
Directory.CreateDirectory(PerfPath);
SNMPEngine = new SNMPEngine();
SNMPEngine.Timeout = 1250;
SNMPEngine = new SNMPEngine();
SNMPEngine.Timeout = 1250;
_CrawlPool = new DiskObject<CrawlPool>(String.Format("{0}/pool",BasePath));
_CrawlPool = new DiskObject<CrawlPool>(String.Format("{0}/pool", BasePath));
crawlThreadPool.NumThreads = 12;
threadScheduler = new Thread(scheduler);
threadScheduler.Start();
threadScheduler = new Thread(scheduler);
threadScheduler.Start();
} catch (Exception)
{
Stop();
throw;
}
}
public void Start()
{
if (CrawlerState == ComponentState.INITIALIZED)
{
stopping = false;
SNMPEngine = new SNMPEngine();
}
}
public void Stop()
@ -80,6 +95,19 @@ namespace ln.skyscanner.crawl
_CrawlPool.Save();
}
public ComponentState CrawlerState
{
get
{
if (stopping)
return ComponentState.STOPPING;
if (crawlThreadPool.CurrentPoolSize == 0)
return ComponentState.INITIALIZED;
return ComponentState.STARTED;
}
}
public void Enqueue(JobDelegate job)
{
crawlThreadPool.Enqueue(job);

12
http/CrawlerApi.cs
View File

@ -5,6 +5,7 @@ using System.Linq;
using ln.skyscanner.crawl;
using ln.types;
using System.Net;
using ln.types.threads;
namespace ln.skyscanner.http
{
@ -51,6 +52,17 @@ namespace ln.skyscanner.http
SkyScanner.Crawler.Crawl(CIDR.Parse(_ip));
}
[Callable]
public PoolJob[] GetCurrentPoolJobs()
{
return SkyScanner.Crawler.CurrentJobs;
}
[Callable]
public PoolJob[] GetQueuedJobs()
{
return SkyScanner.Crawler.QueuedJobs;
}
}
}

20
ln.skyscanner.csproj
View File

@ -61,9 +61,6 @@
<None Include="templates\static\frame.html">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Include="templates\static\style.css">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Include="templates\static\interface.js">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
@ -79,6 +76,21 @@
<None Include="templates\static\tables.js">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Include="templates\static\dist\tabulator.min.js">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Include="templates\static\css\tabulator.min.css">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Include="templates\static\css\style.css">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Include="templates\static\dist\moment-with-locales.js">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Include="templates\static\nav.html">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>
<Folder Include="identify\" />
@ -88,6 +100,8 @@
<Folder Include="http\" />
<Folder Include="templates\" />
<Folder Include="templates\static\" />
<Folder Include="templates\static\dist\" />
<Folder Include="templates\static\css\" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\ln.snmp\ln.snmp.csproj">

205
templates/static/crawler.html
View File

@ -1,92 +1,131 @@
<%frame "frame.html"%>
<div>
<h1>Crawler</h1>
<div class="cpanel">
<button onclick="api.call('api/management','StartCrawler')">Start</button>
<button onclick="api.call('api/management','StopCrawler')">Stop</button>
</div>
<div>
<div class="group right">
<table>
<tr>
<td>Bezeichnung:</td>
<td id="chName"></td>
<td><button onclick="api.call('api/crawler','Crawl', [$('chIP').innerText]);">CRAWL</button></td>
</tr>
<tr>
<td>Prim&auml;re IP:</td>
<td id="chIP"></td>
</tr>
<tr>
<td>Erster Kontakt:</td>
<td id="chFirstSeen"></td>
</tr>
<tr>
<td>Letzter Kontakt:</td>
<td id="chLastSeen"></td>
</tr>
<tr>
<td>N&auml;chster Test:</td>
<td id="chNextCheck"></td>
</tr>
<tr>
<td>Letzter Test:</td>
<td id="chLastCheck"></td>
</tr>
<tr>
<td>Hints:</td>
<td id="chHints"></td>
</tr>
</table>
</div>
<div class="group" style="bottom: 24px;">
<div>Nodes</div>
<table onload="NodeTable(this);"></table>
</div>
</div>
</div>
<script>
function showHostDetails(ip)
<div class="cpanel">
<button onclick="api.call('api/management','StartCrawler')">Start</button>
<button onclick="api.call('api/management','StopCrawler')">Stop</button>
</div>
<div>
<button onclick="refreshHostTable();">!?</button>
<div>
Bezeichnung: <input type="text" id="htName" />
Prim&auml;re IP: <input type="text" id="htIP" />
<button onclick="createNewHost();">+</button>
</div>
</div>
<div class="scroll fill">
<div id="hostTable" onload="createHostTable();" style="height: 400px;"></div>
</div>
<script type="text/javascript">
var hostTable = null;
function createNewHost()
{
var crawledHost = api.call('api/crawler','GetHostByIP', [ip]);
$("chName").innerText = crawledHost.Name;
$("chIP").innerText = crawledHost.PrimaryIP;
$("chFirstSeen").innerText = crawledHost.FirstSeen;
$("chLastSeen").innerText = crawledHost.LastSeen;
$("chLastCheck").innerText = crawledHost.LastCheck;
$("chNextCheck").innerText = crawledHost.NextCheck;
$("chHints").innerText = JSON.stringify( crawledHost.Hints );
}
function NodeTable(t)
{
var columns = [
{ "Label": 'Primäre IP', "Field": "PrimaryIP", "ToAdd": true },
{ "Label": 'Name', "Field": "Name", "ToAdd": true },
{ "Label": 'First Seen', "Field": "FirstSeen" },
{ "Label": 'Last Seen', "Field": "LastSeen" },
{ "Label": 'Last Check', "Field": "LastCheck" },
{ "Label": 'Next Check', "Field": "NextCheck" }
];
var loader = function(){ return api.call('api/crawler','GetHosts'); };
var adder = function(p){
return api.call('api/crawler','AddHost', p);
};
DynamicTable(t,columns,loader, adder);
t.selectedRowChanged = function(row){
showHostDetails(row.PrimaryIP);
try
{
var host = api.call("api/crawler","AddHost", [ $("htIP").value, $("htName").value ]);
hostTable.updateOrAddData( [ host ] );
} catch (e)
{
alert("Exception: " + e);
}
}
function refreshHostTable()
{
var hostList = api.call("api/crawler","GetHosts");
if (!hostList)
hostList = []
hostTable.clearData();
hostTable.addData( hostList );
}
function createHostTable()
{
var columns = [
{
title: "Bezeichnung",
field: "Name",
sorter: "string",
width: 180,
editor: "input"
},
{
title: "Primäre IP",
field: "PrimaryIP",
sorter: "string",
width: 180,
editor: "input",
validator: "required"
},
{
title: "Zuerst gesehen",
field: "FirstSeen",
sorter: "",
width: 180,
formatter: "datetime",
formatterParams: {
inputFormat: "",
outputFormat: "DD.MM.YYYY hh:mm:ss",
invalidPlaceHolder: "-"
}
},
{
title: "Zuletzt gesehen",
field: "LastSeen",
sorter: "",
width: 180,
formatter: "datetime",
formatterParams: {
inputFormat: "",
outputFormat: "DD.MM.YYYY hh:mm:ss",
invalidPlaceHolder: "-"
}
},
{
title: "Letzter Scan",
field: "LastCheck",
sorter: "",
width: 180,
formatter: "datetime",
formatterParams: {
inputFormat: "",
outputFormat: "DD.MM.YYYY hh:mm:ss",
invalidPlaceHolder: "-"
}
},
{
title: "Nächster Scan",
field: "NextCheck",
sorter: "",
width: 180,
formatter: "datetime",
formatterParams: {
inputFormat: "",
outputFormat: "DD.MM.YYYY hh:mm:ss",
invalidPlaceHolder: "-"
}
}
];
var hostList = api.call("api/crawler","GetHosts");
if (!hostList)
hostList = []
hostTable = new Tabulator("#hostTable", {
columns: columns,
data: hostList,
index: "PrimaryIP",
addRowPos: "top"
});
}
</script>

269
templates/static/style.css → templates/static/css/style.css
View File

@ -11,6 +11,21 @@
bottom: 0px;
left: 0px;
right: 0px;
}
div {
margin: 0px;
padding: 0px;
flex-grow: 0;
}
div#body {
display: flex;
height: 100%;
width: 100%;
flex-direction: column;
}
h1,h2,h3,h4,h5,h6 {
@ -18,24 +33,107 @@ h1,h2,h3,h4,h5,h6 {
padding:2px;
}
h1 {
font-size: 18px;
}
h2 {
font-size: 14px;
}
a {
text-decoration: none;
color: inherit;
}
#header {
top: 0px;
left: 0px;
right: 0px;
border-bottom: 1px solid black;
background-color: #c7def5;
padding: 12px;
display: flex;
flex-direction: row;
flex-wrap: wrap;
}
#header > div {
margin: 4px;
}
#footer {
height: 20px;
padding: 6px;
padding-left: 24px;
background-color: #c7def5;
border-top: 1px solid black;
flex-grow: 0;
flex-shrink: 0;
}
#page {
display: flex;
top: 56px;
bottom: 32px;
left: 0px;
right: 0px;
flex-grow: 1;
flex-direction: row;
}
#nav {
display: flex;
left: 0px;
top: 0px;
bottom: 0px;
padding: 6px;
flex-grow: 0;
}
#content {
padding: 8px;
padding-top: 16px;
overflow: auto;
}
.scroll {
display: inline-block;
overflow: scroll;
}
.fill {
flex-grow: 1;
flex-shrink: 1;
}
.right {
float: right;
right: 0px;
}
#header {
border-bottom: 1px solid black;
padding: 6px;
background-color: #c7def5;
.flex {
display: flex;
}
#header > div {
display: inline-block;
.flex.row {
flex-direction: row;
}
.flex > div {
margin: 4px;
}
.skylogo {
@ -43,7 +141,9 @@ a {
border: 1px solid black;
font-size: 18px;
font-weight: 800;
margin-right: 40px;
flex-grow: 0;
flex-shrink: 0;
}
.skylogo > div {
display: inline-block;
@ -83,139 +183,10 @@ a {
.indicator[state="3"] > div {
background-color: #D00000;
}
#footer {
position: absolute;
display: block;
bottom: 0px;
height: 32px;
left: 0px;
right: 0px;
background-color: #c7def5;
padding: 6px;
padding-left: 24px;
.indicator[state="4"] > div {
background-color: #D0D000;
}
#nav {
display: block;
left: 0px;
right: 0px;
text-align: center;
padding: 6px;
}
#nav > div {
}
#nav > div > div {
display: inline-block;
width: 160px;
border: 1px solid black;
padding: 4px;
margin-left: 32px;
margin-right: 32px;
border-radius: 3px;
}
#content {
display: block;
position: absolute;
top: 80px;
bottom: 40px;
left: 0px;
right: 0px;
padding: 12px;
overflow: auto;
}
#workspace {
display: block;
left: 0px;
right: 0px;
top: 0px;
bottom: 0px;
padding-left: 16px;
padding-right: 16px;
}
#workspace > #wstabs {
border-bottom: 1px solid black;
}
#workspace > #wsview {
border: 1px solid black;
border-top: none;
}
#workspace > #wsview > div {
display: block;
position: relative;
left: 0px;
right: 0px;
height: 100%;
}
table {
border-collapse: collapse;
overflow: auto;
height: 600px;
width: 75%;
display: inline-block;
}
table > thead > tr {
}
table > thead > tr > td
{
border-bottom: 1px solid black;
padding-left: 2px;
padding-right: 16px;
font-style: italic;
}
table > tbody {
}
table > tbody > tr {
}
table > tbody > tr > td {
padding-top: 4px;
padding-bottom: 4px;
padding-left: 2px;
padding-right: 16px;
}
table > tbody > tr:hover > td {
background-color: #F0F0D0;
cursor: default;
}
table input {
font-size: 10px;
}
.group {
min-width: 240px;
}
@ -228,3 +199,25 @@ table input {
bottom: 0px;
}
#navigation h1 {
font-size: 14px;
}
#navigation > div {
display: block;
margin-right: 16px;
}
#navigation > div > h1 {
margin-top: 8px;
margin-bottom: 4px;
}
#navigation > div > div {
margin-left: 8px;
}
#content > div {
margin: 6px;
}

3
templates/static/css/tabulator.min.css vendored 100755
View File

File diff suppressed because one or more lines are too long

14765
templates/static/dist/moment-with-locales.js vendored 100644
View File

File diff suppressed because it is too large Load Diff

10
templates/static/dist/tabulator.min.js vendored 100755
View File

File diff suppressed because one or more lines are too long

94
templates/static/frame.html
View File

@ -1,51 +1,45 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<title>SkyScanner WebUI (Alpha)</title>
<link href="/static/style.css" rel="stylesheet" />
<script type="text/javascript" src="/static/interface.js"></script>
<script type="text/javascript" src="/static/tables.js"></script>
</head>
<body onload="initApi('<%=request.self()%>');">
<div id="body">
<section id="header">
<div class="skylogo">
<div style="background-color: #009ee3; color: white;">Sky</div><div style="background-color: white;">Scanner</div>
</div>
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<title>SkyScanner WebUI (Alpha)</title>
<div id="indicators">
<div id="indHttpServer" class="indicator" state="STARTED"><div></div>HTTPServer</div>
<div id="indManager" class="indicator" state=""><div></div>Manager</div>
<div id="indCrawler" class="indicator" state=""><div></div>Crawler</div>
<div id="indChecks" class="indicator" state=""><div></div>Checks</div>
<div id="indDispatcher" class="indicator" state=""><div></div>Message Dispatcher</div>
</div>
<div id="ServerTime"></div>
</section>
<section id="nav">
<div>
<div>
<a href="#" onclick="api.loadPage('static/summary.html'); return false;"><div>&Uuml;bersicht</div></a>
</div>
<div>
<a href="#" onclick="api.loadPage('static/crawler.html'); return false;"><div>Crawler</div></a>
</div>
<div>
<a href="#" onclick="api.loadPage('static/checks.html'); return false;"><div>&Uuml;berwachung</div></a>
</div>
</div>
</section>
<section id="state"></section>
<section id="content">
<%=__frame__%>
</section>
<section id="footer">
<div>
<button onclick="api.call('api/management','Shutdown')">Shutdown</button>
</div>
</section>
</div>
</body>
</html>
<link href="/static/css/style.css" rel="stylesheet" />
<link href="/static/css/tabulator.min.css" rel="stylesheet" />
<script type="text/javascript" src="/static/interface.js"></script>
<script type="text/javascript" src="/static/dist/moment-with-locales.js"></script>
<script type="text/javascript" src="/static/dist/tabulator.min.js"></script>
</head>
<body onload="initApi('<%=request.self()%>');">
<div id="body">
<div id="header">
<div class="skylogo">
<div style="background-color: #009ee3; color: white;">Sky</div><div style="background-color: white;">Scanner</div>
</div>
<div id="indHttpServer" class="indicator" state="STARTED"><div></div>HTTPServer</div>
<div id="indManager" class="indicator" state=""><div></div>Manager</div>
<div id="indCrawler" class="indicator" state=""><div></div>Crawler</div>
<div id="indChecks" class="indicator" state=""><div></div>Checks</div>
<div id="indDispatcher" class="indicator" state=""><div></div>Message Dispatcher</div>
<div id="ServerTime"></div>
</div>
<div id="page">
<div id="nav">
<%include "nav.html"%>
</div>
<div id="content">
<%=__frame__%>
</div>
</div>
<div id="footer">
<div>
<button onclick="api.call('api/management','Shutdown')">Shutdown</button>
</div>
</div>
</div>
</body>
</html>

27
templates/static/nav.html 100644
View File

@ -0,0 +1,27 @@
<div id="navigation">
<div>
<h1>System</h1>
<div>
<a href="#" onclick="api.loadPage('static/summary.html'); return false;"><div>&Uuml;bersicht</div></a>
</div>
</div>
<div>
<h1>Crawler</h1>
<div>
<a href="#" onclick="api.loadPage('static/crawler.html'); return false;"><div>Hosts</div></a>
</div>
</div>
<div>
<h1>Struktur</h1>
<div>
<a href="#" onclick="api.loadPage('static/network.html'); return false;"><div>Übersicht</div></a>
</div>
</div>
<div>
<h1>&Uuml;berwachung</h1>
<div>
<a href="#" onclick="api.loadPage('static/checker.html'); return false;"><div>Hosts</div></a>
</div>
</div>
</div>

64
templates/static/summary.html
View File

@ -1,4 +1,60 @@
<%frame "frame.html"%>
<div>
<h1>&Uuml;bersicht</h1>
</div>
<%frame "frame.html"%>
<div>
<h1>&Uuml;bersicht</h1>
</div>
<div>
<h1>Crawler</h1>
<div class="flex row">
<div>
<h2>Current Jobs</h2>
<div id="crCurrentPoolJobs" onload="createCrawlerTables();" style="height: 600px; width: 400px; "></div>
</div>
<div>
<h2>Queued Jobs</h2>
<div id="crQueuedPoolJobs" style="height: 600px; width: 400px; "></div>
</div>
</div>
</div>
<script type="text/javascript">
var crCurrentPoolJobs = null;
var crQueuedPoolJobs = null;
function refreshHostTable()
{
var hostList = api.call("api/crawler","GetPoolJobs");
if (!hostList)
hostList = []
hostTable.clearData();
hostTable.addData( hostList );
}
function createCrawlerTables()
{
var currentPoolJobs = api.call("api/crawler","GetCurrentPoolJobs");
if (!currentPoolJobs)
currentPoolJobs = [];
var queuedPoolJobs = api.call("api/crawler","GetQueuedJobs");
if (!queuedPoolJobs)
queuedPoolJobs = [];
var columns = [
{ title: "Name", field: "Name" },
{ title: "Progress", field: "Progress" },
{ title: "State", field: "State" },
{ title: "JobState", field: "JobState" }
];
crCurrentPoolJobs = new Tabulator("#crCurrentPoolJobs", {
columns: columns,
data: currentPoolJobs,
});
crQueuedPoolJobs = new Tabulator("#crQueuedPoolJobs", {
columns: columns,
data: queuedPoolJobs,
});
}</script>