WIP

SkyScanner.cs

@@ -9,7 +9,7 @@ using ln.types.threads;
 using System.Threading;
 namespace ln.skyscanner
 {
-    public enum ComponentState { STOPPED, INITIALIZED, STARTED, FAILED }
+    public enum ComponentState { STOPPED, INITIALIZED, STARTED, FAILED, STOPPING }
 
     public class SkyScanner
     {
@@ -50,7 +50,7 @@ namespace ln.skyscanner
             StopCrawler();
             StopHttpServer();
 
-            new Thread(() => ConveniencePool.NumThreads = 0).Start();
+            new Thread(() => ConveniencePool.Close()).Start();
         }
 
 
@@ -101,11 +101,7 @@ namespace ln.skyscanner
             {
                 if (Crawler != null)
                 {
-                    if (Crawler.ThreadStates.Length > 0)
+                    return Crawler.CrawlerState;
-                    {
-                        return ComponentState.STARTED;
-                    }
-                    return ComponentState.INITIALIZED;
                 }
                 return ComponentState.STOPPED;
             }

crawl/Crawler.cs

@@ -38,22 +38,23 @@ namespace ln.skyscanner.crawl
 
         bool stopping;
 
-        Pool crawlThreadPool = new Pool();
+        Pool crawlThreadPool = new Pool(12);
 
-        public PoolThreadState[] ThreadStates => crawlThreadPool.ThreadStates;
+        public PoolJob[] CurrentJobs => crawlThreadPool.CurrentPoolJobs;
-        public int QueuedJobs => crawlThreadPool.QueuedJobs;
+        public PoolJob[] QueuedJobs => crawlThreadPool.QueuedJobs;
 
         public DiskObject<CrawlPool> _CrawlPool;
         public CrawlPool CrawlPool => _CrawlPool.Instance;
 
-        public SNMPEngine SNMPEngine { get; }
+        public SNMPEngine SNMPEngine { get; private set; }
 
         Thread threadScheduler;
 
         public Crawler(SkyScanner skyScanner)
         {
             SkyScanner = skyScanner;
-
+            try
+            {
                 BasePath = Path.Combine(skyScanner.BasePath, "crawler");
 
                 if (!Directory.Exists(BasePath))
@@ -65,11 +66,25 @@ namespace ln.skyscanner.crawl
                 SNMPEngine = new SNMPEngine();
                 SNMPEngine.Timeout = 1250;
 
-            _CrawlPool = new DiskObject<CrawlPool>(String.Format("{0}/pool",BasePath));
+                _CrawlPool = new DiskObject<CrawlPool>(String.Format("{0}/pool", BasePath));
 
-            crawlThreadPool.NumThreads = 12;
                 threadScheduler = new Thread(scheduler);
                 threadScheduler.Start();
+            } catch (Exception)
+            {
+                Stop();
+
+                throw;
+            }
+        }
+
+        public void Start()
+        {
+            if (CrawlerState == ComponentState.INITIALIZED)
+            {
+                stopping = false;
+                SNMPEngine = new SNMPEngine();
+            }
         }
 
         public void Stop()
@@ -80,6 +95,19 @@ namespace ln.skyscanner.crawl
             _CrawlPool.Save();
         }
 
+        public ComponentState CrawlerState
+        {
+            get
+            {
+                if (stopping)
+                    return ComponentState.STOPPING;
+                if (crawlThreadPool.CurrentPoolSize == 0)
+                    return ComponentState.INITIALIZED;
+                return ComponentState.STARTED;
+            }
+        }
+
+
         public void Enqueue(JobDelegate job)
         {
             crawlThreadPool.Enqueue(job);

http/CrawlerApi.cs

@@ -5,6 +5,7 @@ using System.Linq;
 using ln.skyscanner.crawl;
 using ln.types;
 using System.Net;
+using ln.types.threads;
 
 namespace ln.skyscanner.http
 {
@@ -51,6 +52,17 @@ namespace ln.skyscanner.http
             SkyScanner.Crawler.Crawl(CIDR.Parse(_ip));
         }
 
+        [Callable]
+        public PoolJob[] GetCurrentPoolJobs()
+        {
+            return SkyScanner.Crawler.CurrentJobs;
+        }
+
+        [Callable]
+        public PoolJob[] GetQueuedJobs()
+        {
+            return SkyScanner.Crawler.QueuedJobs;
+        }
 
     }
 }

ln.skyscanner.csproj

@@ -61,9 +61,6 @@
     <None Include="templates\static\frame.html">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
-    <None Include="templates\static\style.css">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
     <None Include="templates\static\interface.js">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
@@ -79,6 +76,21 @@
     <None Include="templates\static\tables.js">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Include="templates\static\dist\tabulator.min.js">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Include="templates\static\css\tabulator.min.css">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Include="templates\static\css\style.css">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Include="templates\static\dist\moment-with-locales.js">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Include="templates\static\nav.html">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
   <ItemGroup>
     <Folder Include="identify\" />
@@ -88,6 +100,8 @@
     <Folder Include="http\" />
     <Folder Include="templates\" />
     <Folder Include="templates\static\" />
+    <Folder Include="templates\static\dist\" />
+    <Folder Include="templates\static\css\" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\ln.snmp\ln.snmp.csproj">

templates/static/crawler.html

@@ -1,91 +1,130 @@
 <%frame "frame.html"%>
+
 <div>
     <h1>Crawler</h1>
-    <div class="cpanel">
+</div>
+
+<div class="cpanel">
     <button onclick="api.call('api/management','StartCrawler')">Start</button>
     <button onclick="api.call('api/management','StopCrawler')">Stop</button>
-    </div>
-    
-    <div>
-        <div class="group right">
-            <table>
-                <tr>
-                    <td>Bezeichnung:</td>
-                    <td id="chName"></td>
-                    <td><button onclick="api.call('api/crawler','Crawl', [$('chIP').innerText]);">CRAWL</button></td>
-                </tr>
-                <tr>
-                    <td>Prim&auml;re IP:</td>
-                    <td id="chIP"></td>
-                </tr>
-                <tr>
-                    <td>Erster Kontakt:</td>
-                    <td id="chFirstSeen"></td>
-                </tr>
-                <tr>
-                    <td>Letzter Kontakt:</td>
-                    <td id="chLastSeen"></td>
-                </tr>
-                <tr>
-                    <td>N&auml;chster Test:</td>
-                    <td id="chNextCheck"></td>
-                </tr>
-                <tr>
-                    <td>Letzter Test:</td>
-                    <td id="chLastCheck"></td>
-                </tr>
-                <tr>
-                    <td>Hints:</td>
-                    <td id="chHints"></td>
-                </tr>
-            </table>
-        </div>
-        
-        <div class="group" style="bottom: 24px;">
-            <div>Nodes</div>        
-            <table onload="NodeTable(this);"></table>
-        </div>
-
-    </div>
-
 </div>
-<script>
 
-    function showHostDetails(ip)
+<div>
+    <button onclick="refreshHostTable();">!?</button>
+    <div>
+        Bezeichnung: <input type="text" id="htName" />
+        Prim&auml;re IP: <input type="text" id="htIP" />
+        <button onclick="createNewHost();">+</button>
+    </div>
+</div>
+<div class="scroll fill">
+    <div id="hostTable" onload="createHostTable();" style="height: 400px;"></div>
+</div>
+
+<script type="text/javascript">
+
+    var hostTable = null;
+
+    function createNewHost()
     {
-        var crawledHost = api.call('api/crawler','GetHostByIP', [ip]);
+        try
-        $("chName").innerText = crawledHost.Name;
+        {
-        $("chIP").innerText = crawledHost.PrimaryIP;
+            var host = api.call("api/crawler","AddHost", [  $("htIP").value, $("htName").value ]);
-        $("chFirstSeen").innerText = crawledHost.FirstSeen;
+            hostTable.updateOrAddData( [ host ] );
-        $("chLastSeen").innerText = crawledHost.LastSeen;
+        } catch (e)
-        $("chLastCheck").innerText = crawledHost.LastCheck;
+        {
-        $("chNextCheck").innerText = crawledHost.NextCheck;
+            alert("Exception: " + e);
-        $("chHints").innerText = JSON.stringify( crawledHost.Hints );
+        }
     }
 
+    function refreshHostTable()
+    {
+        var hostList = api.call("api/crawler","GetHosts");
+        if (!hostList)
+            hostList = []
 
-    function NodeTable(t)
+        hostTable.clearData();
+        hostTable.addData( hostList );
+    }
+
+    function createHostTable()
     {
         var columns = [
-            { "Label": 'Primäre IP', "Field": "PrimaryIP", "ToAdd": true },
+            {
-            { "Label": 'Name', "Field": "Name", "ToAdd": true },    
+                title: "Bezeichnung",
-            { "Label": 'First Seen', "Field": "FirstSeen" },    
+                field: "Name",
-            { "Label": 'Last Seen', "Field": "LastSeen" },    
+                sorter: "string",
-            { "Label": 'Last Check', "Field": "LastCheck" },
+                width: 180,
-            { "Label": 'Next Check', "Field": "NextCheck" }
+                editor: "input"
+            },
+            {
+                title: "Primäre IP",
+                field: "PrimaryIP",
+                sorter: "string",
+                width: 180,
+                editor: "input",
+                validator: "required"
+            },
+            {
+                title: "Zuerst gesehen",
+                field: "FirstSeen",
+                sorter: "",
+                width: 180,
+                formatter: "datetime",
+                formatterParams: {
+                    inputFormat: "",
+                    outputFormat: "DD.MM.YYYY hh:mm:ss",
+                    invalidPlaceHolder: "-"
+                }
+            },
+            {
+                title: "Zuletzt gesehen",
+                field: "LastSeen",
+                sorter: "",
+                width: 180,
+                formatter: "datetime",
+                formatterParams: {
+                    inputFormat: "",
+                    outputFormat: "DD.MM.YYYY hh:mm:ss",
+                    invalidPlaceHolder: "-"
+                }
+            },
+            {
+                title: "Letzter Scan",
+                field: "LastCheck",
+                sorter: "",
+                width: 180,
+                formatter: "datetime",
+                formatterParams: {
+                    inputFormat: "",
+                    outputFormat: "DD.MM.YYYY hh:mm:ss",
+                    invalidPlaceHolder: "-"
+                }
+            },
+            {
+                title: "Nächster Scan",
+                field: "NextCheck",
+                sorter: "",
+                width: 180,
+                formatter: "datetime",
+                formatterParams: {
+                    inputFormat: "",
+                    outputFormat: "DD.MM.YYYY hh:mm:ss",
+                    invalidPlaceHolder: "-"
+                }
+            }
         ];
 
-        var loader = function(){ return api.call('api/crawler','GetHosts'); };
+        var hostList = api.call("api/crawler","GetHosts");
-        var adder = function(p){
+        if (!hostList)
-                return api.call('api/crawler','AddHost', p);
+            hostList = []
-            };
 
-        DynamicTable(t,columns,loader, adder);
+        hostTable = new Tabulator("#hostTable", {
-    
+                columns: columns,
-    
+                data: hostList,
-        t.selectedRowChanged = function(row){
+                index: "PrimaryIP",
-            showHostDetails(row.PrimaryIP);
+                addRowPos: "top"
-        }
+            });
     }
 
 

templates/static/css/style.css

@@ -11,6 +11,21 @@
     bottom: 0px;
     left: 0px;
     right: 0px;
+    
+}
+
+div {
+    margin: 0px;
+    padding: 0px;
+    flex-grow: 0;
+}
+
+div#body {
+    display: flex;
+    height: 100%;
+    width: 100%;
+
+    flex-direction: column;
 }
 
 h1,h2,h3,h4,h5,h6 {
@@ -18,24 +33,107 @@ h1,h2,h3,h4,h5,h6 {
     padding:2px;
 }
 
+h1 {
+    font-size: 18px;
+}
+
+h2 {
+    font-size: 14px;
+}
+
 a {
     text-decoration: none;
     color: inherit;
 }
 
+#header {
+    top: 0px;
+    left: 0px;
+    right: 0px;
+
+    border-bottom: 1px solid black;
+    background-color: #c7def5;
+    padding: 12px;
+
+    display: flex;
+
+    flex-direction: row;
+    flex-wrap: wrap;
+}
+
+#header > div {
+    margin: 4px;
+}
+
+
+#footer {
+    height: 20px;
+    padding: 6px;
+    padding-left: 24px;
+    background-color: #c7def5;
+    border-top: 1px solid black;
+
+    flex-grow: 0;
+    flex-shrink: 0;
+}
+
+#page {
+    display: flex;
+    top: 56px;
+    bottom: 32px;
+    left: 0px;
+    right: 0px;
+
+    flex-grow: 1;
+
+    flex-direction: row;
+}
+
+#nav {
+    display: flex;
+
+    left: 0px;
+    top: 0px;
+    bottom: 0px;
+
+    padding: 6px;
+
+    flex-grow: 0;
+}
+
+#content {
+    padding: 8px;
+    padding-top: 16px;
+
+    overflow: auto;
+}
+
+.scroll {
+    display: inline-block;
+    overflow: scroll;
+}
+
+.fill {
+    flex-grow: 1;
+    flex-shrink: 1;
+}
+
 .right {
     float: right;
     right: 0px;
 }
 
-#header {
+.flex {
-    border-bottom: 1px solid black;
+    display: flex;
-    padding: 6px;
+
-    background-color: #c7def5;
 }
 
-#header > div {
+.flex.row {
-    display: inline-block;
+    flex-direction: row;
+}
+
+.flex > div {
+    margin: 4px;
 }
 
 .skylogo {
@@ -43,7 +141,9 @@ a {
     border: 1px solid black;
     font-size: 18px;
     font-weight: 800;
-    margin-right: 40px;
+
+    flex-grow: 0;
+    flex-shrink: 0;
 }
 .skylogo > div {
     display: inline-block;
@@ -83,139 +183,10 @@ a {
 .indicator[state="3"] > div {
     background-color: #D00000;
 }
-
+.indicator[state="4"] > div {
-#footer {
+    background-color: #D0D000;
-    position: absolute;
-    display: block;
-    
-    bottom: 0px;
-    height: 32px;
-    left: 0px;
-    right: 0px;
-    
-
-    background-color: #c7def5;
-    padding: 6px;
-    padding-left: 24px;
 }
 
-#nav {
-    display: block;
-    left: 0px;
-    right: 0px;
-    
-    text-align: center;
-    padding: 6px;
-}
-
-#nav > div {
-}
-
-#nav > div > div {
-    display: inline-block;
-    
-    width: 160px;
-    
-    
-    border: 1px solid black;
-    padding: 4px;
-    
-    margin-left: 32px;
-    margin-right: 32px;
-    
-    
-    border-radius: 3px;
-}
-
-#content {
-    display: block;
-    position: absolute;
-    
-    top: 80px;
-    bottom: 40px;
-    left: 0px;
-    right: 0px;
-    
-    padding: 12px;
-    
-    overflow: auto;
-}
-
-#workspace {
-    display: block;
-
-    left: 0px;
-    right: 0px;
-    top: 0px;
-    bottom: 0px;
-
-    padding-left: 16px;
-    padding-right: 16px;
-}
-
-#workspace > #wstabs {
-    border-bottom: 1px solid black;
-}
-
-#workspace > #wsview {
-    border: 1px solid black;
-    border-top: none;
-}
-
-#workspace > #wsview > div {
-    display: block;
-    position: relative;
-    left: 0px;
-    right: 0px;
-
-    height: 100%;
-}
-
-
-table {
-    border-collapse: collapse;
-    
-    overflow: auto;
-    height: 600px;
-    width: 75%;
-    
-    display: inline-block;
-}
-
-table > thead > tr {
-}
-
-table > thead > tr > td
-{
-    border-bottom: 1px solid black;
-    padding-left: 2px;
-    padding-right: 16px;
-    font-style: italic;
-}
-
-table > tbody {
-}
-
-table > tbody > tr {
-}
-
-table > tbody > tr > td {
-    padding-top: 4px;
-    padding-bottom: 4px;
-    padding-left: 2px;
-    padding-right: 16px;
-}
-
-table > tbody > tr:hover > td {
-    background-color: #F0F0D0;
-    cursor: default;
-}
-
-table input {
-    font-size: 10px;
-}
-
-
 .group {
     min-width: 240px;
 }
@@ -228,3 +199,25 @@ table input {
     bottom: 0px;
 }
 
+#navigation h1 {
+    font-size: 14px;
+}
+
+#navigation > div {
+    display: block;
+    margin-right: 16px;
+}
+
+#navigation > div > h1 {
+    margin-top: 8px;
+    margin-bottom: 4px;
+}
+
+#navigation > div > div {
+    margin-left: 8px;
+}
+
+#content > div {
+    margin: 6px;
+}
+

templates/static/frame.html

@@ -1,51 +1,45 @@
 <!DOCTYPE html>
 <html>
-    <head>
+<head>
     <meta charset="utf-8" />
     <title>SkyScanner WebUI (Alpha)</title>
 
-        <link href="/static/style.css" rel="stylesheet" />
+    <link href="/static/css/style.css" rel="stylesheet" />
+    <link href="/static/css/tabulator.min.css" rel="stylesheet" />
+
     <script type="text/javascript" src="/static/interface.js"></script>
-        <script type="text/javascript" src="/static/tables.js"></script>
+
-    </head>
+    <script type="text/javascript" src="/static/dist/moment-with-locales.js"></script>
-    <body onload="initApi('<%=request.self()%>');">
+    <script type="text/javascript" src="/static/dist/tabulator.min.js"></script>
+</head>
+<body onload="initApi('<%=request.self()%>');">
     <div id="body">
-            <section id="header">
+        <div id="header">
             <div class="skylogo">
                 <div style="background-color: #009ee3; color: white;">Sky</div><div style="background-color: white;">Scanner</div>
             </div>
-
-                <div id="indicators">
             <div id="indHttpServer" class="indicator" state="STARTED"><div></div>HTTPServer</div>
             <div id="indManager" class="indicator" state=""><div></div>Manager</div>
             <div id="indCrawler" class="indicator" state=""><div></div>Crawler</div>
             <div id="indChecks" class="indicator" state=""><div></div>Checks</div>
             <div id="indDispatcher" class="indicator" state=""><div></div>Message Dispatcher</div>
-                </div>
             <div id="ServerTime"></div>
-            </section>
-            <section id="nav">
-                <div>
-                <div>
-                    <a href="#" onclick="api.loadPage('static/summary.html'); return false;"><div>&Uuml;bersicht</div></a>
         </div>
-                <div>
+
-                    <a href="#" onclick="api.loadPage('static/crawler.html'); return false;"><div>Crawler</div></a>
+        <div id="page">
+            <div id="nav">
+                <%include "nav.html"%>
             </div>
-                <div>
+            <div id="content">
-                    <a href="#" onclick="api.loadPage('static/checks.html'); return false;"><div>&Uuml;berwachung</div></a>
-                </div>
-                </div>
-            </section>
-            <section id="state"></section>
-            <section id="content">
                 <%=__frame__%>
-            </section>
+            </div>
-            <section id="footer">
+        </div>
+
+        <div id="footer">
             <div>
                 <button onclick="api.call('api/management','Shutdown')">Shutdown</button>
             </div>
-            </section>
         </div>
-    </body>
+    </div>
+</body>
 </html>

templates/static/nav.html

@@ -0,0 +1,27 @@
+<div id="navigation">
+    <div>
+        <h1>System</h1>
+        <div>
+            <a href="#" onclick="api.loadPage('static/summary.html'); return false;"><div>&Uuml;bersicht</div></a>
+        </div>
+    </div>
+    <div>
+        <h1>Crawler</h1>
+        <div>
+            <a href="#" onclick="api.loadPage('static/crawler.html'); return false;"><div>Hosts</div></a>
+        </div>
+    </div>
+    <div>
+        <h1>Struktur</h1>
+        <div>
+            <a href="#" onclick="api.loadPage('static/network.html'); return false;"><div>Übersicht</div></a>
+        </div>
+    </div>
+    <div>
+        <h1>&Uuml;berwachung</h1>
+        <div>
+            <a href="#" onclick="api.loadPage('static/checker.html'); return false;"><div>Hosts</div></a>
+        </div>
+    </div>
+
+</div>

templates/static/summary.html

@@ -2,3 +2,59 @@
 <div>
     <h1>&Uuml;bersicht</h1>
 </div>
+
+<div>
+    <h1>Crawler</h1>
+    <div class="flex row">
+        <div>
+            <h2>Current Jobs</h2>
+            <div id="crCurrentPoolJobs" onload="createCrawlerTables();" style="height: 600px; width: 400px; "></div>
+        </div>
+        <div>
+            <h2>Queued Jobs</h2>
+            <div id="crQueuedPoolJobs" style="height: 600px; width: 400px; "></div>
+        </div>
+    </div>
+</div>
+
+
+<script type="text/javascript">
+    var crCurrentPoolJobs = null;
+    var crQueuedPoolJobs = null;
+
+    function refreshHostTable()
+    {
+        var hostList = api.call("api/crawler","GetPoolJobs");
+        if (!hostList)
+            hostList = []
+
+        hostTable.clearData();
+        hostTable.addData( hostList );
+    }
+
+    function createCrawlerTables()
+    {
+        var currentPoolJobs = api.call("api/crawler","GetCurrentPoolJobs");
+        if (!currentPoolJobs)
+            currentPoolJobs = [];
+
+        var queuedPoolJobs = api.call("api/crawler","GetQueuedJobs");
+        if (!queuedPoolJobs)
+            queuedPoolJobs = [];
+
+        var columns = [
+            { title: "Name", field: "Name" },
+            { title: "Progress", field: "Progress" },
+            { title: "State", field: "State" },
+            { title: "JobState", field: "JobState" }
+        ];
+
+        crCurrentPoolJobs = new Tabulator("#crCurrentPoolJobs", {
+                columns: columns,
+                data: currentPoolJobs,
+            });
+        crQueuedPoolJobs = new Tabulator("#crQueuedPoolJobs", {
+                columns: columns,
+                data: queuedPoolJobs,
+            });
+    }</script>