From 93abb73b586ca7f7ec3d80d16070543a06daef24 Mon Sep 17 00:00:00 2001 From: Harald Wolff Date: Sun, 29 Nov 2020 22:02:16 +0100 Subject: [PATCH] CIService, RepositoryInterface --- ln.build.server/Program.cs | 69 +----------- ln.build.server/ln.build.server.csproj | 2 +- ln.build/CIJob.cs | 42 ++++--- ln.build/CIService.cs | 77 ++++++++++++- ln.build/ln.build.csproj | 1 + .../repositories/GitRepositoryInterface.cs | 28 +++++ .../repositories/GiteaRepositoryInterface.cs | 105 ++++++++++++++++++ ln.build/repositories/RepositoryInterface.cs | 18 +++ 8 files changed, 257 insertions(+), 85 deletions(-) create mode 100644 ln.build/repositories/GitRepositoryInterface.cs create mode 100644 ln.build/repositories/GiteaRepositoryInterface.cs create mode 100644 ln.build/repositories/RepositoryInterface.cs diff --git a/ln.build.server/Program.cs b/ln.build.server/Program.cs index e8f2bfd..185e59f 100644 --- a/ln.build.server/Program.cs +++ b/ln.build.server/Program.cs @@ -7,6 +7,7 @@ using ln.logging; using ln.type; using ln.threading; using ln.build; +using ln.build.repositories; namespace ln.build.server { @@ -14,77 +15,15 @@ namespace ln.build.server { static CIService CIService; - static HttpResponse WebHookRequest(HttpRoutingContext context, HttpRequest request) - { - HttpResponse response = new HttpResponse(request); - - if (!request.Method.Equals("POST")) - { - response.StatusCode = 405; - } else if (!request.GetRequestHeader("content-type").Equals("application/json")) - { - response.StatusCode = 415; - response.ContentWriter.WriteLine("Unsupported Media Type, should be application/json"); - } else { - JSONValue jsonRequest = JSONParser.Parse(request.ContentReader.ReadToEnd()); - if (jsonRequest is JSONObject message) - { - try - { - string cloneUrl = message["repository"]["clone_url"].ToNative().ToString(); - - foreach (JSONValue commit in (message["commits"] as JSONArray).Children) - { - string commitID = commit["id"].ToNative().ToString(); - Logging.Log("Received CI request of repository {0} for commit {1}", cloneUrl, commitID); - - CIJob job = new CIJob(cloneUrl) - .SetVariable("REPO_OWNER", message["repository"]["owner"]["username"].ToNative().ToString()) - .SetVariable("REPO_NAME", message["repository"]["name"].ToNative().ToString()) - .SetVariable("COMMIT_ID", commitID) - .SetVariable("NOTIFY", message["pusher"]["email"].ToNative().ToString()) - .SetVariable("NUGET_APIKEY", "3yAJPMxcaEhb_HP62dxK") - .SetVariable("NUGET_SOURCE", "http://nuget.l--n.de/nuget/l--n/v3/index.json") - ; - - job.UpdateBuildState(BuildState.PENDING); - - CIService.Enqueue(job); - } - - } catch (Exception e) - { - response.StatusCode = 500; - response.StatusMessage = "An exception occured"; - response.ContentWriter.WriteLine("{0}", e.ToString()); - Logging.Log(e); - } - - } else { - response.StatusCode = 400; - } - - } - - - return response; - } static void Main(string[] args) { CIService = new CIService(); - CIService.Start(); - + CIService.Initialize(); CIService.AddPipeline(new DotNetPipeLine()); + CIService.AddRepositoryInterface(new GiteaRepositoryInterface("https://git.l--n.de")); - - SimpleRouter genericRouter = new SimpleRouter(); - genericRouter.AddSimpleRoute("/", WebHookRequest); - - HTTPServer httpServer = new HTTPServer(new Endpoint(IPv6.ANY, 1888), new LoggingRouter(genericRouter)); - - Logging.Log("Starting http listener..."); - httpServer.Start(); + CIService.Start(); } diff --git a/ln.build.server/ln.build.server.csproj b/ln.build.server/ln.build.server.csproj index ff604be..69f41ae 100644 --- a/ln.build.server/ln.build.server.csproj +++ b/ln.build.server/ln.build.server.csproj @@ -16,7 +16,7 @@ - + diff --git a/ln.build/CIJob.cs b/ln.build/CIJob.cs index 4af0a56..3dbc4c6 100644 --- a/ln.build/CIJob.cs +++ b/ln.build/CIJob.cs @@ -1,17 +1,13 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Net.Http; -using System.Reflection; -using System.Reflection.Metadata.Ecma335; -using System.Security.Cryptography.X509Certificates; using System.Text; using ln.build.commands; +using ln.build.repositories; using ln.json; using ln.logging; using ln.threading; -using Microsoft.VisualBasic; namespace ln.build { @@ -28,8 +24,11 @@ namespace ln.build public CommandEnvironment Environment { get; } + public RepositoryInterface RepositoryInterface { get; set; } + List pipeLines = new List(); + public BuildState BuildState { get; private set; } Dictionary logStreams = new Dictionary(); Dictionary logStreamLoggers = new Dictionary(); @@ -65,6 +64,14 @@ namespace ln.build return logStreamLogger; } + public string GetLoggedContent(string name) + { + logStreams[name].Position = 0; + using StreamReader sr = new StreamReader(logStreams[name]); + return sr.ReadToEnd(); + } + public IEnumerable GetLogStreamNames() => logStreams.Keys; + public string GetVariable(string varName) => Environment.Get(varName); public string GetVariable(string varName,string defValue) => Environment.Get(varName, defValue); public bool ContainsVariable(string varName) => Environment.Contains(varName); @@ -72,6 +79,8 @@ namespace ln.build public async void UpdateBuildState(BuildState buildState) { + BuildState = buildState; + string buildStateURL = String.Format("https://git.l--n.de/api/v1/repos/{0}/{1}/statuses/{2}", GetVariable("REPO_OWNER"), GetVariable("REPO_NAME"), @@ -89,13 +98,9 @@ namespace ln.build HttpResponseMessage response = httpClient.PostAsync(buildStateURL, new StringContent(stateObject.ToString(),Encoding.UTF8,"application/json")).Result; Logger.Log(LogLevel.DEBUG, "UpdateBuildState({0}): {1}", buildState, buildStateURL); Logger.Log(LogLevel.DEBUG, "Response: {0}", response ); - } } - - - public override void RunJob() { if (CloneRepository()) @@ -146,20 +151,21 @@ namespace ln.build public void Notify() { - foreach (string key in logStreams.Keys) - { - logStreams[key].Position = 0; - string logContent; - using (StreamReader sr = new StreamReader(logStreams[key])) - logContent = sr.ReadToEnd(); + // foreach (string key in logStreams.Keys) + // { + // logStreams[key].Position = 0; + // string logContent; + // using (StreamReader sr = new StreamReader(logStreams[key])) + // logContent = sr.ReadToEnd(); - Logger.Log(LogLevel.INFO, "------------------- LogStream {0} ---------------------------\n{1}", key, logContent); - } + // Logger.Log(LogLevel.INFO, "------------------- LogStream {0} ---------------------------\n{1}", key, logContent); + // } } public void Cleanup() { - //Directory.Delete(WorkingDirectory, true); + CurrentCIService.CreateReport(this); + Directory.Delete(WorkingDirectory, true); } diff --git a/ln.build/CIService.cs b/ln.build/CIService.cs index 1e0db4d..1de019e 100644 --- a/ln.build/CIService.cs +++ b/ln.build/CIService.cs @@ -1,12 +1,29 @@ +using System; using System.Collections.Generic; +using System.IO; +using ln.build.repositories; +using ln.http; +using ln.http.client; +using ln.http.router; +using ln.json; +using ln.logging; using ln.threading; +using ln.type; namespace ln.build { public class CIService { - public Pool buildPool; + public string ReportsDirectory { get; set; } = "./builds"; + public Endpoint HttpEndpoint { get; } = new Endpoint(IPv6.ANY, 1888); + + Logger webLogger; + SimpleRouter httpRouter; + SimpleRouter hookRouter; + HTTPServer httpServer; + + List repositoryInterfaces = new List(); HashSet pipelines = new HashSet(); public IEnumerable PipeLines => pipelines; @@ -14,11 +31,34 @@ namespace ln.build public CIService() { buildPool = new Pool(2); + ReportsDirectory = Path.GetFullPath(ReportsDirectory); + } + + public void Initialize() + { + Directory.CreateDirectory(ReportsDirectory); + + InitializeHttpServer(); + } + + private void InitializeHttpServer() + { + httpRouter = new SimpleRouter(); + httpRouter.AddSimpleRoute("/builds/*", new StaticRouter(ReportsDirectory)); + + hookRouter = new SimpleRouter(); + httpRouter.AddSimpleRoute("/hooks/*", hookRouter); + + webLogger = new Logger(Logger.ConsoleLogger); + webLogger.Backends.Add(new FileLogger("ln.build.http.log")); + + httpServer = new HTTPServer(HttpEndpoint, new LoggingRouter(httpRouter, webLogger)); } public void Start() { buildPool.Start(); + httpServer.Start(); } public void AddPipeline(PipeLine pipeLine) @@ -26,12 +66,47 @@ namespace ln.build pipelines.Add(pipeLine); } + public void AddRepositoryInterface(RepositoryInterface repositoryInterface) + { + hookRouter.AddSimpleRoute(String.Format("/{0}", repositoryInterface.WebHookName), (HttpRoutingContext context,HttpRequest request) => repositoryInterface.WebHookHandler(this, request)); + repositoryInterfaces.Add(repositoryInterface); + } + + public void Enqueue(CIJob job) { job.CurrentCIService = this; buildPool.Enqueue(job); } + public void CreateReport(CIJob job) + { + JSONObject report = new JSONObject(); + JSONArray steps = new JSONArray(); + report["steps"] = steps; + report["state"] = new JSONString(job.BuildState.ToString()); + + string reportDirectory = Path.Combine(ReportsDirectory, job.JobID); + Directory.CreateDirectory(reportDirectory); + + foreach (string logName in job.GetLogStreamNames()) + { + string logFileName = Path.Combine(reportDirectory, logName); + using (FileStream fileStream = new FileStream(logFileName, FileMode.CreateNew)) + { + job.GetLogStream(logName).Position = 0; + job.GetLogStream(logName).CopyTo(fileStream); + fileStream.Flush(); + fileStream.Close(); + } + + steps.Add(new JSONString(logName)); + } + + using (StreamWriter writer = new StreamWriter(Path.Combine(reportDirectory, "build.json"))) + writer.Write(report.ToString()); + } + } diff --git a/ln.build/ln.build.csproj b/ln.build/ln.build.csproj index 6979766..4ddffb2 100644 --- a/ln.build/ln.build.csproj +++ b/ln.build/ln.build.csproj @@ -18,6 +18,7 @@ + diff --git a/ln.build/repositories/GitRepositoryInterface.cs b/ln.build/repositories/GitRepositoryInterface.cs new file mode 100644 index 0000000..045f063 --- /dev/null +++ b/ln.build/repositories/GitRepositoryInterface.cs @@ -0,0 +1,28 @@ + +using System; +using ln.build.commands; +using ln.http; + +namespace ln.build.repositories +{ + public abstract class GitRepositoryInterface : RepositoryInterface + { + string webHookName; + public override string WebHookName => webHookName; + + public GitRepositoryInterface(string webHookName) + { + this.webHookName = webHookName; + } + + public override void CloneSources(CIJob job) + { + job.Logger.Log("{0}: cloning repository to {1}", GetType().Name, job.WorkingDirectory); + bool success = (new CommandRunner("git", "clone", job.RepositoryURL, job.WorkingDirectory).Run(job.Environment) == 0) && + (!job.ContainsVariable("COMMIT_ID") || new CommandRunner("git", "checkout", job.GetVariable("COMMIT_ID")){ WorkingDirectory = job.WorkingDirectory, }.Run(job.Environment) == 0); + if (!success) + throw new Exception("clone failed"); + } + + } +} \ No newline at end of file diff --git a/ln.build/repositories/GiteaRepositoryInterface.cs b/ln.build/repositories/GiteaRepositoryInterface.cs new file mode 100644 index 0000000..85eaf7e --- /dev/null +++ b/ln.build/repositories/GiteaRepositoryInterface.cs @@ -0,0 +1,105 @@ +using System; +using System.Net.Http; +using System.Text; +using ln.http; +using ln.json; +using ln.logging; + +namespace ln.build.repositories +{ + + public class GiteaRepositoryInterface : GitRepositoryInterface + { + public string BaseURL { get; } + + public GiteaRepositoryInterface(string baseURL) :base("gitea") + { + BaseURL = baseURL; + } + + public override bool DetectValidity(string cloneUrl) => cloneUrl.StartsWith(BaseURL); + public override void UpdateBuildState(CIJob job) + { + string buildStateURL = string.Format("{3}/api/v1/repos/{0}/{1}/statuses/{2}", + job.GetVariable("REPO_OWNER"), + job.GetVariable("REPO_NAME"), + job.GetVariable("COMMIT_ID"), + BaseURL + ); + + if (buildStateURL != null) + { + JSONObject stateObject = new JSONObject(); + stateObject.Add("context", "ln.build"); + stateObject.Add("description", "build job pending"); + stateObject.Add("state", job.BuildState.ToString().ToLower()); + stateObject.Add("target_url", JSONNull.Instance); + + using (HttpClient httpClient = new HttpClient()) + { + HttpResponseMessage response = httpClient.PostAsync(buildStateURL, new StringContent(stateObject.ToString(),Encoding.UTF8,"application/json")).Result; + job.Logger.Log(LogLevel.DEBUG, "UpdateBuildState({0}): {1}", job.BuildState, buildStateURL); + job.Logger.Log(LogLevel.DEBUG, "Response: {0}", response ); + } + } + } + + public override HttpResponse WebHookHandler(CIService ciService, HttpRequest request) + { + HttpResponse response = new HttpResponse(request); + + if (!request.Method.Equals("POST")) + { + response.StatusCode = 405; + } else if (!request.GetRequestHeader("content-type").Equals("application/json")) + { + response.StatusCode = 415; + response.ContentWriter.WriteLine("Unsupported Media Type, should be application/json"); + } else { + JSONValue jsonRequest = JSONParser.Parse(request.ContentReader.ReadToEnd()); + if (jsonRequest is JSONObject message) + { + try + { + string cloneUrl = message["repository"]["clone_url"].ToNative().ToString(); + + foreach (JSONValue commit in (message["commits"] as JSONArray).Children) + { + string commitID = commit["id"].ToNative().ToString(); + Logging.Log("Received CI request of repository {0} for commit {1}", cloneUrl, commitID); + + CIJob job = new CIJob(cloneUrl) + .SetVariable("REPO_OWNER", message["repository"]["owner"]["username"].ToNative().ToString()) + .SetVariable("REPO_NAME", message["repository"]["name"].ToNative().ToString()) + .SetVariable("COMMIT_ID", commitID) + .SetVariable("NOTIFY", message["pusher"]["email"].ToNative().ToString()) + .SetVariable("NUGET_APIKEY", "3yAJPMxcaEhb_HP62dxK") + .SetVariable("NUGET_SOURCE", "http://nuget.l--n.de/nuget/l--n/v3/index.json") + ; + + job.UpdateBuildState(BuildState.PENDING); + + ciService.Enqueue(job); + } + + } catch (Exception e) + { + response.StatusCode = 500; + response.StatusMessage = "An exception occured"; + response.ContentWriter.WriteLine("{0}", e.ToString()); + Logging.Log(e); + } + + } else { + response.StatusCode = 400; + } + + } + + + return response; + + } + } + +} \ No newline at end of file diff --git a/ln.build/repositories/RepositoryInterface.cs b/ln.build/repositories/RepositoryInterface.cs new file mode 100644 index 0000000..8291f27 --- /dev/null +++ b/ln.build/repositories/RepositoryInterface.cs @@ -0,0 +1,18 @@ + +using System; +using ln.http; + +namespace ln.build.repositories +{ + public abstract class RepositoryInterface + { + + public abstract String WebHookName { get; } + public abstract HttpResponse WebHookHandler(CIService service, HttpRequest request); + + public abstract bool DetectValidity(string cloneUrl); + public abstract void CloneSources(CIJob job); + public abstract void UpdateBuildState(CIJob job); + + } +} \ No newline at end of file