From 8455a2dad45b75a7a9fe9642d1c55532f54219ac Mon Sep 17 00:00:00 2001 From: Harald Wolff Date: Sat, 28 Nov 2020 01:08:49 +0100 Subject: [PATCH] Several Fixes and improvements. --- ln.build.server/Program.cs | 29 ++++----- ln.build.server/ln.build.server.csproj | 2 +- ln.build/CIJob.cs | 88 ++++++++++++++++---------- ln.build/CIService.cs | 38 +++++++++++ ln.build/CommandRunner.cs | 9 ++- ln.build/DotNetPipeLine.cs | 15 ++++- ln.build/PipeLine.cs | 5 +- ln.build/ln.build.csproj | 2 +- 8 files changed, 131 insertions(+), 57 deletions(-) create mode 100644 ln.build/CIService.cs diff --git a/ln.build.server/Program.cs b/ln.build.server/Program.cs index 9402a42..97f8724 100644 --- a/ln.build.server/Program.cs +++ b/ln.build.server/Program.cs @@ -13,8 +13,7 @@ namespace ln.build.server class Program { - static Pool pool = new Pool(2); - + static CIService CIService; static HttpResponse WebHookRequest(HttpRoutingContext context, HttpRequest request) { HttpResponse response = new HttpResponse(request); @@ -32,28 +31,24 @@ namespace ln.build.server { try { - string repoName = message["repository"]["name"].ToNative().ToString(); string cloneUrl = message["repository"]["clone_url"].ToNative().ToString(); - string notifyEmail = message["pusher"]["email"].ToNative().ToString(); foreach (JSONValue commit in (message["commits"] as JSONArray).Children) { string commitID = commit["id"].ToNative().ToString(); + Logging.Log("Received CI request of repository {0} for commit {1}", cloneUrl, commitID); - Logging.Log("Received CI request for repository {2} [{0}] for the commit {1}", cloneUrl, commitID, repoName); - - CIJob job = new CIJob(repoName, cloneUrl, commitID, notifyEmail); - - job.SetVariable("REPO_OWNER", message["repository"]["owner"]["username"].ToNative().ToString()); - job.SetVariable("REPO_NAME", message["repository"]["name"].ToNative().ToString()); - job.SetVariable("COMMIT_ID", commitID); + CIJob job = new CIJob(cloneUrl) + .SetVariable("REPO_OWNER", message["repository"]["owner"]["username"].ToNative().ToString()) + .SetVariable("REPO_NAME", message["repository"]["name"].ToNative().ToString()) + .SetVariable("COMMIT_ID", commitID) + .SetVariable("NOTIFY", message["pusher"]["email"].ToNative().ToString()); job.UpdateBuildState(BuildState.PENDING); - pool.Enqueue(job); + CIService.Enqueue(job); } - } catch (Exception e) { response.StatusCode = 500; @@ -74,15 +69,17 @@ namespace ln.build.server static void Main(string[] args) { - CommandRunner.DefaultThrow = CRThrow.NEGATIVE; + CIService = new CIService(); + CIService.Start(); + CIService.AddPipeline(new DotNetPipeLine()); + + SimpleRouter genericRouter = new SimpleRouter(); genericRouter.AddSimpleRoute("/", WebHookRequest); HTTPServer httpServer = new HTTPServer(new Endpoint(IPv6.ANY, 1888), new LoggingRouter(genericRouter)); - pool.Start(); - Logging.Log("Starting http listener..."); httpServer.Start(); } diff --git a/ln.build.server/ln.build.server.csproj b/ln.build.server/ln.build.server.csproj index 76fa324..ff604be 100644 --- a/ln.build.server/ln.build.server.csproj +++ b/ln.build.server/ln.build.server.csproj @@ -6,7 +6,7 @@ - 0.1.0-test + 0.1.0-test1 Harald Wolff-Thobaben l--n.de A simple build server scheduling builds triggered via web-hooks diff --git a/ln.build/CIJob.cs b/ln.build/CIJob.cs index a76b61d..977ac8f 100644 --- a/ln.build/CIJob.cs +++ b/ln.build/CIJob.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using System.Net.Http; using System.Reflection; +using System.Reflection.Metadata.Ecma335; using System.Security.Cryptography.X509Certificates; using System.Text; using ln.json; @@ -16,15 +17,12 @@ namespace ln.build { static HttpClient httpClient = new HttpClient(); - public string JobID { get; } = Guid.NewGuid().ToString("N"); public string RepositoryURL { get; } - public string RepositoryName { get; } - public string Commit { get; } - public string NotifyEMail { get; set; } public string WorkingDirectory { get; set; } + public CIService CurrentCIService { get; set; } public Logger Logger { get; private set; } @@ -34,16 +32,13 @@ namespace ln.build Dictionary logStreams = new Dictionary(); Dictionary logStreamLoggers = new Dictionary(); - public CIJob(string repositoryName, string repositoryURL, string commit, string notifyEMail) + public CIJob(string repositoryURL) { WorkingDirectory = Path.Combine(Path.GetTempPath(), JobID); Logger = new Logger(new FileLogger(Path.Combine(Path.GetTempPath(), String.Format("{0}.log", JobID)))); Logger.Backends.Add(Logger.ConsoleLogger); - RepositoryName = repositoryName; RepositoryURL = repositoryURL; - Commit = commit; - NotifyEMail = notifyEMail; } public Stream GetLogStream(string name) @@ -73,12 +68,14 @@ namespace ln.build value = defValue; return value; } - public void SetVariable(string varName,string value) + public CIJob SetVariable(string varName,string value) { if (value != null) variables[varName] = value; else variables.Remove(varName); + + return this; } public void ExtendVariable(string varName,string value) => ExtendVariable(varName, value, ':'); public void ExtendVariable(string varName, string value, char seperator) @@ -93,6 +90,16 @@ namespace ln.build SetVariable(varName, currentValue); } + public bool ContainsVariable(string varName) => variables.ContainsKey(varName); + public bool ContainsVariable(params string[] varNames) + { + foreach (string varName in varNames) + { + if (!variables.ContainsKey(varName)) + return false; + } + return true; + } public async void UpdateBuildState(BuildState buildState) { @@ -110,7 +117,7 @@ namespace ln.build stateObject.Add("state", buildState.ToString().ToLower()); stateObject.Add("target_url", JSONNull.Instance); - HttpResponseMessage response = await httpClient.PostAsync(buildStateURL, new StringContent(stateObject.ToString(),Encoding.UTF8,"application/json")); + HttpResponseMessage response = httpClient.PostAsync(buildStateURL, new StringContent(stateObject.ToString(),Encoding.UTF8,"application/json")).Result; Logger.Log(LogLevel.DEBUG, "UpdateBuildState({0}): {1}", buildState, buildStateURL); Logger.Log(LogLevel.DEBUG, "Response: {0}", response ); @@ -122,39 +129,50 @@ namespace ln.build public override void RunJob() { - CloneRepository(); - DetectPipelines(); - - try{ - foreach (PipeLine pipeLine in pipeLines) - { - pipeLine.Run(this); - } - UpdateBuildState(BuildState.SUCCESS); - } catch (Exception e) + if (CloneRepository()) { - UpdateBuildState(BuildState.FAILURE); + if (DetectPipelines()) + { + UpdateBuildState(BuildState.PENDING); + + try{ + foreach (PipeLine pipeLine in pipeLines) + { + pipeLine.Run(this); + } + UpdateBuildState(BuildState.SUCCESS); + } catch (Exception e) + { + UpdateBuildState(BuildState.FAILURE); + } + } + } else { + Logger.Log(LogLevel.ERROR, "CIJob failed at CloneRepository()"); } - + Notify(); Cleanup(); } - void DetectPipelines() - { - string[] sln = Directory.GetFileSystemEntries(WorkingDirectory,"*.sln"); - if (sln.Length > 0) - { - pipeLines.Add(new DotNetPipeLine()); - } - } - - - public void CloneRepository() + public bool CloneRepository() { Logging.Log("cloning repository to {0}", WorkingDirectory); - new CommandRunner(Logger, "git", "clone",RepositoryURL,WorkingDirectory).Run(); - new CommandRunner(Logger, "git", "checkout", Commit).Run(); + return + (new CommandRunner(Logger, "git", "clone", RepositoryURL, WorkingDirectory).Run() == 0) && + (!ContainsVariable("COMMIT_ID") || new CommandRunner(Logger, "git", "checkout", GetVariable("COMMIT_ID")){ WorkingDirectory = this.WorkingDirectory, }.Run() == 0); + } + + public bool DetectPipelines() + { + if (CurrentCIService != null) + { + foreach (PipeLine pipeLine in CurrentCIService.PipeLines) + { + if (pipeLine.DetectValidity(this)) + pipeLines.Add(pipeLine); + } + } + return pipeLines.Count > 0; } public void Notify() diff --git a/ln.build/CIService.cs b/ln.build/CIService.cs new file mode 100644 index 0000000..1e0db4d --- /dev/null +++ b/ln.build/CIService.cs @@ -0,0 +1,38 @@ +using System.Collections.Generic; +using ln.threading; + +namespace ln.build +{ + public class CIService + { + + public Pool buildPool; + + HashSet pipelines = new HashSet(); + public IEnumerable PipeLines => pipelines; + + public CIService() + { + buildPool = new Pool(2); + } + + public void Start() + { + buildPool.Start(); + } + + public void AddPipeline(PipeLine pipeLine) + { + pipelines.Add(pipeLine); + } + + public void Enqueue(CIJob job) + { + job.CurrentCIService = this; + buildPool.Enqueue(job); + } + + + + } +} \ No newline at end of file diff --git a/ln.build/CommandRunner.cs b/ln.build/CommandRunner.cs index d1bc4d1..bfca27a 100644 --- a/ln.build/CommandRunner.cs +++ b/ln.build/CommandRunner.cs @@ -6,7 +6,6 @@ using ln.logging; namespace ln.build { public enum CRThrow { NEVER, NEGATIVE, NONZERO } - public class CommandRunner { public static CRThrow DefaultThrow { get; set; } = CRThrow.NONZERO; @@ -18,12 +17,17 @@ namespace ln.build public Logger Logger { get; } + Func TestExitCode = null; + public CommandRunner(string executable,params string[] arguments) : this(Logger.Default, executable, arguments){} - public CommandRunner(Logger logger, string executable,params string[] arguments) + public CommandRunner(Logger logger, string executable,params string[] arguments) : this(logger, null, executable, arguments){} + public CommandRunner(Logger logger, Func testExitCode, string executable,params string[] arguments) { Logger = logger; Executable = executable; Arguments = arguments; + + TestExitCode = testExitCode; } public string FindFileInPath(string filename) @@ -68,6 +72,7 @@ namespace ln.build logger.Log(LogLevel.INFO, "Result: {0}", process.ExitCode); if ( + ((TestExitCode != null) && !TestExitCode(process.ExitCode)) || ((Throw == CRThrow.NEGATIVE) && (process.ExitCode < 0)) || ((Throw == CRThrow.NONZERO) && (process.ExitCode != 0)) ) diff --git a/ln.build/DotNetPipeLine.cs b/ln.build/DotNetPipeLine.cs index 5eb5957..1d4c35d 100644 --- a/ln.build/DotNetPipeLine.cs +++ b/ln.build/DotNetPipeLine.cs @@ -2,6 +2,7 @@ using System; using System.IO; using System.Reflection.Metadata.Ecma335; +using System.Runtime.InteropServices.WindowsRuntime; using System.Runtime.Serialization.Formatters; using System.Text; using System.Text.RegularExpressions; @@ -33,7 +34,19 @@ namespace ln.build AddStep(new PublishStep()); } - + + public override bool DetectValidity(CIJob job) + { + string[] sln = Directory.GetFileSystemEntries(job.WorkingDirectory,"*.sln"); + if (sln.Length > 0) + return true; + sln = Directory.GetFileSystemEntries(job.WorkingDirectory,"*.csproj"); + if (sln.Length > 0) + return true; + + return false; + } + void FilterPackageFilenames(CommandStep commandStep, CIJob job, int exitCode) { Stream outStream = job.GetLogStream(commandStep.DefaultLogFileName); diff --git a/ln.build/PipeLine.cs b/ln.build/PipeLine.cs index 74624e5..e36b129 100644 --- a/ln.build/PipeLine.cs +++ b/ln.build/PipeLine.cs @@ -8,7 +8,7 @@ using ln.logging; namespace ln.build { - public class PipeLine + public abstract class PipeLine { List steps = new List(); @@ -17,6 +17,9 @@ namespace ln.build { } + public abstract bool DetectValidity(CIJob job); + + public void AddStep(Step step) => steps.Add(step); public virtual void Run(CIJob job) diff --git a/ln.build/ln.build.csproj b/ln.build/ln.build.csproj index 2d2e5a2..6979766 100644 --- a/ln.build/ln.build.csproj +++ b/ln.build/ln.build.csproj @@ -5,7 +5,7 @@ - 0.1.0 + 0.1.0-test1 Harald Wolff-Thobaben l--n.de A simple build server scheduling builds triggered via web-hooks