ln-dotnet
/
ln.build
3
0
Fork 0
Code Issues 2 Pull Requests Releases 5 Wiki Activity

CIService, RepositoryInterface
ln.build build job pending

master
Harald Wolff 2020-11-29 22:02:16 +01:00
parent 4ddade2c32
commit 93abb73b58
8 changed files with 257 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
ln.build.server/Program.cs
View File

@ -7,6 +7,7 @@ using ln.logging;
using ln.type;
using ln.threading;
using ln.build;
using ln.build.repositories;
namespace ln.build.server
{
@ -14,77 +15,15 @@ namespace ln.build.server
{
static CIService CIService;
static HttpResponse WebHookRequest(HttpRoutingContext context, HttpRequest request)
{
HttpResponse response = new HttpResponse(request);
if (!request.Method.Equals("POST"))
{
response.StatusCode = 405;
} else if (!request.GetRequestHeader("content-type").Equals("application/json"))
{
response.StatusCode = 415;
response.ContentWriter.WriteLine("Unsupported Media Type, should be application/json");
} else {
JSONValue jsonRequest = JSONParser.Parse(request.ContentReader.ReadToEnd());
if (jsonRequest is JSONObject message)
{
try
{
string cloneUrl = message["repository"]["clone_url"].ToNative().ToString();
foreach (JSONValue commit in (message["commits"] as JSONArray).Children)
{
string commitID = commit["id"].ToNative().ToString();
Logging.Log("Received CI request of repository {0} for commit {1}", cloneUrl, commitID);
CIJob job = new CIJob(cloneUrl)
.SetVariable("REPO_OWNER", message["repository"]["owner"]["username"].ToNative().ToString())
.SetVariable("REPO_NAME", message["repository"]["name"].ToNative().ToString())
.SetVariable("COMMIT_ID", commitID)
.SetVariable("NOTIFY", message["pusher"]["email"].ToNative().ToString())
.SetVariable("NUGET_APIKEY", "3yAJPMxcaEhb_HP62dxK")
.SetVariable("NUGET_SOURCE", "http://nuget.l--n.de/nuget/l--n/v3/index.json")
;
job.UpdateBuildState(BuildState.PENDING);
CIService.Enqueue(job);
}
} catch (Exception e)
{
response.StatusCode = 500;
response.StatusMessage = "An exception occured";
response.ContentWriter.WriteLine("{0}", e.ToString());
Logging.Log(e);
}
} else {
response.StatusCode = 400;
}
}
return response;
}
static void Main(string[] args)
{
CIService = new CIService();
CIService.Start();
CIService.Initialize();
CIService.AddPipeline(new DotNetPipeLine());
CIService.AddRepositoryInterface(new GiteaRepositoryInterface("https://git.l--n.de"));
SimpleRouter genericRouter = new SimpleRouter();
genericRouter.AddSimpleRoute("/", WebHookRequest);
HTTPServer httpServer = new HTTPServer(new Endpoint(IPv6.ANY, 1888), new LoggingRouter(genericRouter));
Logging.Log("Starting http listener...");
httpServer.Start();
CIService.Start();
}

2
ln.build.server/ln.build.server.csproj
View File

@ -16,7 +16,7 @@
<ItemGroup>
<PackageReference Include="ln.application" Version="0.1.1" />
<PackageReference Include="ln.http" Version="0.1.1" />
<PackageReference Include="ln.http" Version="0.1.2" />
<PackageReference Include="ln.json" Version="1.0.0" />
<PackageReference Include="ln.logging" Version="1.0.1" />
<PackageReference Include="ln.threading" Version="0.1.0" />

42
ln.build/CIJob.cs
View File

@ -1,17 +1,13 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Reflection;
using System.Reflection.Metadata.Ecma335;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using ln.build.commands;
using ln.build.repositories;
using ln.json;
using ln.logging;
using ln.threading;
using Microsoft.VisualBasic;
namespace ln.build
{
@ -28,8 +24,11 @@ namespace ln.build
public CommandEnvironment Environment { get; }
public RepositoryInterface RepositoryInterface { get; set; }
List<PipeLine> pipeLines = new List<PipeLine>();
public BuildState BuildState { get; private set; }
Dictionary<string,MemoryStream> logStreams = new Dictionary<string, MemoryStream>();
Dictionary<string,Logger> logStreamLoggers = new Dictionary<string, Logger>();
@ -65,6 +64,14 @@ namespace ln.build
return logStreamLogger;
}
public string GetLoggedContent(string name)
{
logStreams[name].Position = 0;
using StreamReader sr = new StreamReader(logStreams[name]);
return sr.ReadToEnd();
}
public IEnumerable<string> GetLogStreamNames() => logStreams.Keys;
public string GetVariable(string varName) => Environment.Get(varName);
public string GetVariable(string varName,string defValue) => Environment.Get(varName, defValue);
public bool ContainsVariable(string varName) => Environment.Contains(varName);
@ -72,6 +79,8 @@ namespace ln.build
public async void UpdateBuildState(BuildState buildState)
{
BuildState = buildState;
string buildStateURL = String.Format("https://git.l--n.de/api/v1/repos/{0}/{1}/statuses/{2}",
GetVariable("REPO_OWNER"),
GetVariable("REPO_NAME"),
@ -89,13 +98,9 @@ namespace ln.build
HttpResponseMessage response = httpClient.PostAsync(buildStateURL, new StringContent(stateObject.ToString(),Encoding.UTF8,"application/json")).Result;
Logger.Log(LogLevel.DEBUG, "UpdateBuildState({0}): {1}", buildState, buildStateURL);
Logger.Log(LogLevel.DEBUG, "Response: {0}", response );
}
}
public override void RunJob()
{
if (CloneRepository())
@ -146,20 +151,21 @@ namespace ln.build
public void Notify()
{
foreach (string key in logStreams.Keys)
{
logStreams[key].Position = 0;
string logContent;
using (StreamReader sr = new StreamReader(logStreams[key]))
logContent = sr.ReadToEnd();
// foreach (string key in logStreams.Keys)
// {
// logStreams[key].Position = 0;
// string logContent;
// using (StreamReader sr = new StreamReader(logStreams[key]))
// logContent = sr.ReadToEnd();
Logger.Log(LogLevel.INFO, "------------------- LogStream {0} ---------------------------\n{1}", key, logContent);
}
// Logger.Log(LogLevel.INFO, "------------------- LogStream {0} ---------------------------\n{1}", key, logContent);
// }
}
public void Cleanup()
{
//Directory.Delete(WorkingDirectory, true);
CurrentCIService.CreateReport(this);
Directory.Delete(WorkingDirectory, true);
}

77
ln.build/CIService.cs
View File

@ -1,12 +1,29 @@
using System;
using System.Collections.Generic;
using System.IO;
using ln.build.repositories;
using ln.http;
using ln.http.client;
using ln.http.router;
using ln.json;
using ln.logging;
using ln.threading;
using ln.type;
namespace ln.build
{
public class CIService
{
public Pool buildPool;
public string ReportsDirectory { get; set; } = "./builds";
public Endpoint HttpEndpoint { get; } = new Endpoint(IPv6.ANY, 1888);
Logger webLogger;
SimpleRouter httpRouter;
SimpleRouter hookRouter;
HTTPServer httpServer;
List<RepositoryInterface> repositoryInterfaces = new List<RepositoryInterface>();
HashSet<PipeLine> pipelines = new HashSet<PipeLine>();
public IEnumerable<PipeLine> PipeLines => pipelines;
@ -14,11 +31,34 @@ namespace ln.build
public CIService()
{
buildPool = new Pool(2);
ReportsDirectory = Path.GetFullPath(ReportsDirectory);
}
public void Initialize()
{
Directory.CreateDirectory(ReportsDirectory);
InitializeHttpServer();
}
private void InitializeHttpServer()
{
httpRouter = new SimpleRouter();
httpRouter.AddSimpleRoute("/builds/*", new StaticRouter(ReportsDirectory));
hookRouter = new SimpleRouter();
httpRouter.AddSimpleRoute("/hooks/*", hookRouter);
webLogger = new Logger(Logger.ConsoleLogger);
webLogger.Backends.Add(new FileLogger("ln.build.http.log"));
httpServer = new HTTPServer(HttpEndpoint, new LoggingRouter(httpRouter, webLogger));
}
public void Start()
{
buildPool.Start();
httpServer.Start();
}
public void AddPipeline(PipeLine pipeLine)
@ -26,12 +66,47 @@ namespace ln.build
pipelines.Add(pipeLine);
}
public void AddRepositoryInterface(RepositoryInterface repositoryInterface)
{
hookRouter.AddSimpleRoute(String.Format("/{0}", repositoryInterface.WebHookName), (HttpRoutingContext context,HttpRequest request) => repositoryInterface.WebHookHandler(this, request));
repositoryInterfaces.Add(repositoryInterface);
}
public void Enqueue(CIJob job)
{
job.CurrentCIService = this;
buildPool.Enqueue(job);
}
public void CreateReport(CIJob job)
{
JSONObject report = new JSONObject();
JSONArray steps = new JSONArray();
report["steps"] = steps;
report["state"] = new JSONString(job.BuildState.ToString());
string reportDirectory = Path.Combine(ReportsDirectory, job.JobID);
Directory.CreateDirectory(reportDirectory);
foreach (string logName in job.GetLogStreamNames())
{
string logFileName = Path.Combine(reportDirectory, logName);
using (FileStream fileStream = new FileStream(logFileName, FileMode.CreateNew))
{
job.GetLogStream(logName).Position = 0;
job.GetLogStream(logName).CopyTo(fileStream);
fileStream.Flush();
fileStream.Close();
}
steps.Add(new JSONString(logName));
}
using (StreamWriter writer = new StreamWriter(Path.Combine(reportDirectory, "build.json")))
writer.Write(report.ToString());
}
}

1
ln.build/ln.build.csproj
View File

@ -18,6 +18,7 @@
<PackageReference Include="ln.logging" Version="1.0.1" />
<PackageReference Include="ln.threading" Version="0.1.0" />
<PackageReference Include="ln.json" Version="1.0.0" />
<PackageReference Include="ln.http" Version="0.1.2" />
</ItemGroup>
</Project>

28
ln.build/repositories/GitRepositoryInterface.cs 100644
View File

@ -0,0 +1,28 @@
using System;
using ln.build.commands;
using ln.http;
namespace ln.build.repositories
{
public abstract class GitRepositoryInterface : RepositoryInterface
{
string webHookName;
public override string WebHookName => webHookName;
public GitRepositoryInterface(string webHookName)
{
this.webHookName = webHookName;
}
public override void CloneSources(CIJob job)
{
job.Logger.Log("{0}: cloning repository to {1}", GetType().Name, job.WorkingDirectory);
bool success = (new CommandRunner("git", "clone", job.RepositoryURL, job.WorkingDirectory).Run(job.Environment) == 0) &&
(!job.ContainsVariable("COMMIT_ID") || new CommandRunner("git", "checkout", job.GetVariable("COMMIT_ID")){ WorkingDirectory = job.WorkingDirectory, }.Run(job.Environment) == 0);
if (!success)
throw new Exception("clone failed");
}
}
}

105
ln.build/repositories/GiteaRepositoryInterface.cs 100644
View File

@ -0,0 +1,105 @@
using System;
using System.Net.Http;
using System.Text;
using ln.http;
using ln.json;
using ln.logging;
namespace ln.build.repositories
{
public class GiteaRepositoryInterface : GitRepositoryInterface
{
public string BaseURL { get; }
public GiteaRepositoryInterface(string baseURL) :base("gitea")
{
BaseURL = baseURL;
}
public override bool DetectValidity(string cloneUrl) => cloneUrl.StartsWith(BaseURL);
public override void UpdateBuildState(CIJob job)
{
string buildStateURL = string.Format("{3}/api/v1/repos/{0}/{1}/statuses/{2}",
job.GetVariable("REPO_OWNER"),
job.GetVariable("REPO_NAME"),
job.GetVariable("COMMIT_ID"),
BaseURL
);
if (buildStateURL != null)
{
JSONObject stateObject = new JSONObject();
stateObject.Add("context", "ln.build");
stateObject.Add("description", "build job pending");
stateObject.Add("state", job.BuildState.ToString().ToLower());
stateObject.Add("target_url", JSONNull.Instance);
using (HttpClient httpClient = new HttpClient())
{
HttpResponseMessage response = httpClient.PostAsync(buildStateURL, new StringContent(stateObject.ToString(),Encoding.UTF8,"application/json")).Result;
job.Logger.Log(LogLevel.DEBUG, "UpdateBuildState({0}): {1}", job.BuildState, buildStateURL);
job.Logger.Log(LogLevel.DEBUG, "Response: {0}", response );
}
}
}
public override HttpResponse WebHookHandler(CIService ciService, HttpRequest request)
{
HttpResponse response = new HttpResponse(request);
if (!request.Method.Equals("POST"))
{
response.StatusCode = 405;
} else if (!request.GetRequestHeader("content-type").Equals("application/json"))
{
response.StatusCode = 415;
response.ContentWriter.WriteLine("Unsupported Media Type, should be application/json");
} else {
JSONValue jsonRequest = JSONParser.Parse(request.ContentReader.ReadToEnd());
if (jsonRequest is JSONObject message)
{
try
{
string cloneUrl = message["repository"]["clone_url"].ToNative().ToString();
foreach (JSONValue commit in (message["commits"] as JSONArray).Children)
{
string commitID = commit["id"].ToNative().ToString();
Logging.Log("Received CI request of repository {0} for commit {1}", cloneUrl, commitID);
CIJob job = new CIJob(cloneUrl)
.SetVariable("REPO_OWNER", message["repository"]["owner"]["username"].ToNative().ToString())
.SetVariable("REPO_NAME", message["repository"]["name"].ToNative().ToString())
.SetVariable("COMMIT_ID", commitID)
.SetVariable("NOTIFY", message["pusher"]["email"].ToNative().ToString())
.SetVariable("NUGET_APIKEY", "3yAJPMxcaEhb_HP62dxK")
.SetVariable("NUGET_SOURCE", "http://nuget.l--n.de/nuget/l--n/v3/index.json")
;
job.UpdateBuildState(BuildState.PENDING);
ciService.Enqueue(job);
}
} catch (Exception e)
{
response.StatusCode = 500;
response.StatusMessage = "An exception occured";
response.ContentWriter.WriteLine("{0}", e.ToString());
Logging.Log(e);
}
} else {
response.StatusCode = 400;
}
}
return response;
}
}
}

18
ln.build/repositories/RepositoryInterface.cs 100644
View File

@ -0,0 +1,18 @@
using System;
using ln.http;
namespace ln.build.repositories
{
public abstract class RepositoryInterface
{
public abstract String WebHookName { get; }
public abstract HttpResponse WebHookHandler(CIService service, HttpRequest request);
public abstract bool DetectValidity(string cloneUrl);
public abstract void CloneSources(CIJob job);
public abstract void UpdateBuildState(CIJob job);
}
}