From 182d0be12a0e5169d312001e0f7415c53f8b9082 Mon Sep 17 00:00:00 2001 From: Harald Wolff Date: Fri, 27 Nov 2020 22:12:01 +0100 Subject: [PATCH] Initial Commit --- .gitignore | 41 ++++++ ln.build.server/Program.cs | 93 +++++++++++++ ln.build.server/ln.build.server.csproj | 29 ++++ ln.build.sln | 48 +++++++ ln.build/BuildState.cs | 6 + ln.build/CIJob.cs | 175 +++++++++++++++++++++++++ ln.build/CommandRunner.cs | 127 ++++++++++++++++++ ln.build/DotNetPipeLine.cs | 99 ++++++++++++++ ln.build/PipeLine.cs | 73 +++++++++++ ln.build/ln.build.csproj | 23 ++++ 10 files changed, 714 insertions(+) create mode 100644 .gitignore create mode 100644 ln.build.server/Program.cs create mode 100644 ln.build.server/ln.build.server.csproj create mode 100644 ln.build.sln create mode 100644 ln.build/BuildState.cs create mode 100644 ln.build/CIJob.cs create mode 100644 ln.build/CommandRunner.cs create mode 100644 ln.build/DotNetPipeLine.cs create mode 100644 ln.build/PipeLine.cs create mode 100644 ln.build/ln.build.csproj diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bf793ed --- /dev/null +++ b/.gitignore @@ -0,0 +1,41 @@ +# Autosave files +*~ + +# build +[Oo]bj/ +[Bb]in/ +packages/ +TestResults/ + +# globs +Makefile.in +*.DS_Store +*.sln.cache +*.suo +*.cache +*.pidb +*.userprefs +*.usertasks +config.log +config.make +config.status +aclocal.m4 +install-sh +autom4te.cache/ +*.user +*.tar.gz +tarballs/ +test-results/ +Thumbs.db +.vs/ + +# Mac bundle stuff +*.dmg +*.app + +# resharper +*_Resharper.* +*.Resharper + +# dotCover +*.dotCover diff --git a/ln.build.server/Program.cs b/ln.build.server/Program.cs new file mode 100644 index 0000000..9402a42 --- /dev/null +++ b/ln.build.server/Program.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using ln.http; +using ln.http.router; +using ln.json; +using ln.logging; +using ln.type; +using ln.threading; +using ln.build; + +namespace ln.build.server +{ + class Program + { + + static Pool pool = new Pool(2); + + static HttpResponse WebHookRequest(HttpRoutingContext context, HttpRequest request) + { + HttpResponse response = new HttpResponse(request); + + if (!request.Method.Equals("POST")) + { + response.StatusCode = 405; + } else if (!request.GetRequestHeader("content-type").Equals("application/json")) + { + response.StatusCode = 415; + response.ContentWriter.WriteLine("Unsupported Media Type, should be application/json"); + } else { + JSONValue jsonRequest = JSONParser.Parse(request.ContentReader.ReadToEnd()); + if (jsonRequest is JSONObject message) + { + try + { + string repoName = message["repository"]["name"].ToNative().ToString(); + string cloneUrl = message["repository"]["clone_url"].ToNative().ToString(); + string notifyEmail = message["pusher"]["email"].ToNative().ToString(); + + foreach (JSONValue commit in (message["commits"] as JSONArray).Children) + { + string commitID = commit["id"].ToNative().ToString(); + + Logging.Log("Received CI request for repository {2} [{0}] for the commit {1}", cloneUrl, commitID, repoName); + + CIJob job = new CIJob(repoName, cloneUrl, commitID, notifyEmail); + + job.SetVariable("REPO_OWNER", message["repository"]["owner"]["username"].ToNative().ToString()); + job.SetVariable("REPO_NAME", message["repository"]["name"].ToNative().ToString()); + job.SetVariable("COMMIT_ID", commitID); + + job.UpdateBuildState(BuildState.PENDING); + + pool.Enqueue(job); + } + + + } catch (Exception e) + { + response.StatusCode = 500; + response.StatusMessage = "An exception occured"; + response.ContentWriter.WriteLine("{0}", e.ToString()); + Logging.Log(e); + } + + } else { + response.StatusCode = 400; + } + + } + + + return response; + } + + static void Main(string[] args) + { + CommandRunner.DefaultThrow = CRThrow.NEGATIVE; + + SimpleRouter genericRouter = new SimpleRouter(); + genericRouter.AddSimpleRoute("/", WebHookRequest); + + HTTPServer httpServer = new HTTPServer(new Endpoint(IPv6.ANY, 1888), new LoggingRouter(genericRouter)); + + pool.Start(); + + Logging.Log("Starting http listener..."); + httpServer.Start(); + } + + + + } +} diff --git a/ln.build.server/ln.build.server.csproj b/ln.build.server/ln.build.server.csproj new file mode 100644 index 0000000..539cb2f --- /dev/null +++ b/ln.build.server/ln.build.server.csproj @@ -0,0 +1,29 @@ + + + + Exe + netcoreapp3.1 + + + + 0.1.0 + Harald Wolff-Thobaben + l--n.de + A simple build server scheduling builds triggered via web-hooks + (c) 2020 Harald Wolff-Thobaben + build build-server + + + + + + + + + + + + + + + diff --git a/ln.build.sln b/ln.build.sln new file mode 100644 index 0000000..48d4e10 --- /dev/null +++ b/ln.build.sln @@ -0,0 +1,48 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.26124.0 +MinimumVisualStudioVersion = 15.0.26124.0 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ln.build", "ln.build\ln.build.csproj", "{682A1FD5-3722-4C16-BED9-FFD90ED40FE8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ln.build.server", "ln.build.server\ln.build.server.csproj", "{3437B6AB-7937-42DD-A526-5716E0114C61}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Debug|x64.ActiveCfg = Debug|Any CPU + {682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Debug|x64.Build.0 = Debug|Any CPU + {682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Debug|x86.ActiveCfg = Debug|Any CPU + {682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Debug|x86.Build.0 = Debug|Any CPU + {682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Release|Any CPU.Build.0 = Release|Any CPU + {682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Release|x64.ActiveCfg = Release|Any CPU + {682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Release|x64.Build.0 = Release|Any CPU + {682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Release|x86.ActiveCfg = Release|Any CPU + {682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Release|x86.Build.0 = Release|Any CPU + {3437B6AB-7937-42DD-A526-5716E0114C61}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3437B6AB-7937-42DD-A526-5716E0114C61}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3437B6AB-7937-42DD-A526-5716E0114C61}.Debug|x64.ActiveCfg = Debug|Any CPU + {3437B6AB-7937-42DD-A526-5716E0114C61}.Debug|x64.Build.0 = Debug|Any CPU + {3437B6AB-7937-42DD-A526-5716E0114C61}.Debug|x86.ActiveCfg = Debug|Any CPU + {3437B6AB-7937-42DD-A526-5716E0114C61}.Debug|x86.Build.0 = Debug|Any CPU + {3437B6AB-7937-42DD-A526-5716E0114C61}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3437B6AB-7937-42DD-A526-5716E0114C61}.Release|Any CPU.Build.0 = Release|Any CPU + {3437B6AB-7937-42DD-A526-5716E0114C61}.Release|x64.ActiveCfg = Release|Any CPU + {3437B6AB-7937-42DD-A526-5716E0114C61}.Release|x64.Build.0 = Release|Any CPU + {3437B6AB-7937-42DD-A526-5716E0114C61}.Release|x86.ActiveCfg = Release|Any CPU + {3437B6AB-7937-42DD-A526-5716E0114C61}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/ln.build/BuildState.cs b/ln.build/BuildState.cs new file mode 100644 index 0000000..742aded --- /dev/null +++ b/ln.build/BuildState.cs @@ -0,0 +1,6 @@ +namespace ln.build +{ + +public enum BuildState { PENDING, SUCCESS, ERROR, FAILURE, WARNING } + +} \ No newline at end of file diff --git a/ln.build/CIJob.cs b/ln.build/CIJob.cs new file mode 100644 index 0000000..a76b61d --- /dev/null +++ b/ln.build/CIJob.cs @@ -0,0 +1,175 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Reflection; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using ln.json; +using ln.logging; +using ln.threading; + +namespace ln.build +{ + public class CIJob : PoolJob + { + static HttpClient httpClient = new HttpClient(); + + + public string JobID { get; } = Guid.NewGuid().ToString("N"); + + public string RepositoryURL { get; } + public string RepositoryName { get; } + public string Commit { get; } + public string NotifyEMail { get; set; } + + public string WorkingDirectory { get; set; } + + public Logger Logger { get; private set; } + + List pipeLines = new List(); + Dictionary variables = new Dictionary(); + + Dictionary logStreams = new Dictionary(); + Dictionary logStreamLoggers = new Dictionary(); + + public CIJob(string repositoryName, string repositoryURL, string commit, string notifyEMail) + { + WorkingDirectory = Path.Combine(Path.GetTempPath(), JobID); + Logger = new Logger(new FileLogger(Path.Combine(Path.GetTempPath(), String.Format("{0}.log", JobID)))); + Logger.Backends.Add(Logger.ConsoleLogger); + + RepositoryName = repositoryName; + RepositoryURL = repositoryURL; + Commit = commit; + NotifyEMail = notifyEMail; + } + + public Stream GetLogStream(string name) + { + if (!logStreams.TryGetValue(name,out MemoryStream logStream)) + { + logStream = new MemoryStream(); + logStreams.Add(name, logStream); + } + return logStream; + } + public Logger GetLogger(string name) + { + if (!logStreamLoggers.TryGetValue(name, out Logger logStreamLogger)) + { + logStreamLogger = new Logger(GetLogStream(name)); + logStreamLogger.Backends.Add(Logger.ConsoleLogger); + logStreamLoggers.Add(name, logStreamLogger); + } + return logStreamLogger; + } + + public string GetVariable(string varName) => GetVariable(varName, null); + public string GetVariable(string varName,string defValue) + { + if (!variables.TryGetValue(varName,out string value)) + value = defValue; + return value; + } + public void SetVariable(string varName,string value) + { + if (value != null) + variables[varName] = value; + else + variables.Remove(varName); + } + public void ExtendVariable(string varName,string value) => ExtendVariable(varName, value, ':'); + public void ExtendVariable(string varName, string value, char seperator) + { + String currentValue = GetVariable(varName, ""); + if (String.Empty.Equals(currentValue)) + { + currentValue = value; + } else { + currentValue = string.Format("{0}{1}{2}",currentValue, seperator, value); + } + SetVariable(varName, currentValue); + } + + + public async void UpdateBuildState(BuildState buildState) + { + string buildStateURL = String.Format("https://git.l--n.de/api/v1/repos/{0}/{1}/statuses/{2}", + GetVariable("REPO_OWNER"), + GetVariable("REPO_NAME"), + GetVariable("COMMIT_ID") + ); + + if (buildStateURL != null) + { + JSONObject stateObject = new JSONObject(); + stateObject.Add("context", "ln.build"); + stateObject.Add("description", "build job pending"); + stateObject.Add("state", buildState.ToString().ToLower()); + stateObject.Add("target_url", JSONNull.Instance); + + HttpResponseMessage response = await httpClient.PostAsync(buildStateURL, new StringContent(stateObject.ToString(),Encoding.UTF8,"application/json")); + Logger.Log(LogLevel.DEBUG, "UpdateBuildState({0}): {1}", buildState, buildStateURL); + Logger.Log(LogLevel.DEBUG, "Response: {0}", response ); + + } + } + + + + + public override void RunJob() + { + CloneRepository(); + DetectPipelines(); + + try{ + foreach (PipeLine pipeLine in pipeLines) + { + pipeLine.Run(this); + } + UpdateBuildState(BuildState.SUCCESS); + } catch (Exception e) + { + UpdateBuildState(BuildState.FAILURE); + } + + Notify(); + Cleanup(); + } + + void DetectPipelines() + { + string[] sln = Directory.GetFileSystemEntries(WorkingDirectory,"*.sln"); + if (sln.Length > 0) + { + pipeLines.Add(new DotNetPipeLine()); + } + } + + + public void CloneRepository() + { + Logging.Log("cloning repository to {0}", WorkingDirectory); + new CommandRunner(Logger, "git", "clone",RepositoryURL,WorkingDirectory).Run(); + new CommandRunner(Logger, "git", "checkout", Commit).Run(); + } + + public void Notify() + { + } + + public void Cleanup() + { + Directory.Delete(WorkingDirectory, true); + } + + + static CIJob() + { + httpClient.DefaultRequestHeaders.Add("Authorization","token 1d03e9577c404b5b4f46b340147b1d500ff95b2e"); + } + } +} diff --git a/ln.build/CommandRunner.cs b/ln.build/CommandRunner.cs new file mode 100644 index 0000000..d1bc4d1 --- /dev/null +++ b/ln.build/CommandRunner.cs @@ -0,0 +1,127 @@ +using System; +using System.Diagnostics; +using System.IO; +using ln.logging; + +namespace ln.build +{ + public enum CRThrow { NEVER, NEGATIVE, NONZERO } + + public class CommandRunner + { + public static CRThrow DefaultThrow { get; set; } = CRThrow.NONZERO; + + public CRThrow Throw { get; set; } = DefaultThrow; + public string Executable { get; } + public string[] Arguments { get; set; } + public string WorkingDirectory { get; set; } = "."; + + public Logger Logger { get; } + + public CommandRunner(string executable,params string[] arguments) : this(Logger.Default, executable, arguments){} + public CommandRunner(Logger logger, string executable,params string[] arguments) + { + Logger = logger; + Executable = executable; + Arguments = arguments; + } + + public string FindFileInPath(string filename) + { + Logger.Log(LogLevel.DEBUG, "Looking up {0} in paths {1}", filename, Environment.GetEnvironmentVariable("PATH")); + + foreach (string path in Environment.GetEnvironmentVariable("PATH").Split(Path.PathSeparator)) + { + string fullpath = Path.Combine(path,filename); + if (File.Exists(fullpath)) + return fullpath; + } + + return filename; + } + + public int Run() => Run(Logger, null, null); + public int Run(Stream stdout, Stream stderr) => Run(Logger, stdout, stderr); + public int Run(Logger logger, Stream stdout) => Run(logger, stdout, stdout); + public int Run(Logger logger, Stream stdout, Stream stderr) + { + ProcessStartInfo psi = new ProcessStartInfo(FindFileInPath(Executable), string.Join(' ', Arguments)) + { + CreateNoWindow = true, + RedirectStandardError = true, + RedirectStandardOutput = true, + RedirectStandardInput = true, + WorkingDirectory = this.WorkingDirectory + }; + psi.EnvironmentVariables.Remove("LANG"); + + logger.Log(LogLevel.INFO, "Executing: {0} {1}", psi.FileName, psi.Arguments); + + Process process = Process.Start(psi); + process.WaitForExit(); + + if (stdout != null) + process.StandardOutput.BaseStream.CopyTo(stdout); + if (stderr != null) + process.StandardError.BaseStream.CopyTo(stderr); + + logger.Log(LogLevel.INFO, "Result: {0}", process.ExitCode); + + if ( + ((Throw == CRThrow.NEGATIVE) && (process.ExitCode < 0)) || + ((Throw == CRThrow.NONZERO) && (process.ExitCode != 0)) + ) + throw new Exception(String.Format("{0} execution gave result {1}", psi.FileName, process.ExitCode)); + + return process.ExitCode; + } + + public int Run(out string stdout,out string stderr) => Run(Logger, out stdout, out stderr); + public int Run(Logger logger, out string stdout,out string stderr) + { + MemoryStream outstream,errstream; + + outstream = new MemoryStream(); + errstream = new MemoryStream(); + + int status = Run(logger, outstream, errstream); + + using (StreamReader sr = new StreamReader(outstream)) + stdout = sr.ReadToEnd(); + + using (StreamReader sr = new StreamReader(errstream)) + stderr = sr.ReadToEnd(); + + return status; + } + + + + + + + +/* + public static int Run(string executable,out Stream stdout,out Stream stderr,params string[] arguments) + { + CommandRunner runner = new CommandRunner(executable,arguments); + return runner.Run(out stdout,out stderr); + } + + public static int Run(string executable,out string stdout,out string stderr,params string[] arguments) => Run(Logger.Default, executable, out stdout, out stderr, arguments); + public static int Run(Logger logger, string executable, out string stdout, out string stderr, params string[] arguments) + { + CommandRunner runner = new CommandRunner(logger, executable, arguments); + int result = runner.Run(out stdout, out stderr); + + return result; + } + + public static int Run(string executable,params string[] arguments) => Run(executable,out Stream stdout,out Stream stderr,arguments); + public static int Run(Logger logger, string executable, params string[] arguments){ + + return Run(executable,out Stream stdout,out Stream stderr,arguments); + } +*/ + } +} \ No newline at end of file diff --git a/ln.build/DotNetPipeLine.cs b/ln.build/DotNetPipeLine.cs new file mode 100644 index 0000000..5eb5957 --- /dev/null +++ b/ln.build/DotNetPipeLine.cs @@ -0,0 +1,99 @@ + +using System; +using System.IO; +using System.Reflection.Metadata.Ecma335; +using System.Runtime.Serialization.Formatters; +using System.Text; +using System.Text.RegularExpressions; +using LibGit2Sharp.Handlers; +using ln.logging; +using ln.type; + +namespace ln.build +{ + + public class DotNetPipeLine : PipeLine + { + static Regex regexPackages = new Regex(".*(Successfully created package '(.*)').*"); + + public DotNetPipeLine() + { + AddStep(new RestoreStep()); + AddStep(new CleanStep()); + + BuildStep buildStep = new BuildStep(); + buildStep.OnCommandExited += FilterPackageFilenames; + AddStep(buildStep); + + AddStep(new TestStep()); + + PackStep packStep = new PackStep(); + packStep.OnCommandExited += FilterPackageFilenames; + AddStep(packStep); + + AddStep(new PublishStep()); + } + + void FilterPackageFilenames(CommandStep commandStep, CIJob job, int exitCode) + { + Stream outStream = job.GetLogStream(commandStep.DefaultLogFileName); + outStream.Position = 0; + + job.Logger.Log(LogLevel.INFO,"filterPackageFilenames(): started"); + + String consoleOutput = Encoding.UTF8.GetString(outStream.ReadToEnd()); + + MatchCollection matches = regexPackages.Matches(consoleOutput); + foreach (Match match in matches) + { + Logging.Log(LogLevel.INFO,"filterPackageFilenames(): {0}", match.Groups[2].Value); + job.ExtendVariable("DOTNET_PACKAGES", match.Groups[2].Value); + } + } + + + class CleanStep : CommandStep + { + public CleanStep():base("clean", "dotnet","clean"){ } + } + class RestoreStep : CommandStep + { + public RestoreStep():base("restore", "dotnet","restore"){ } + } + class TestStep : CommandStep + { + public TestStep():base("test", "dotnet","test"){ } + } + class BuildStep : CommandStep + { + public BuildStep():base("build", "dotnet","build"){ } + } + class PackStep : CommandStep + { + public PackStep():base("pack", "dotnet","pack"){ } + } + class PublishStep : CommandStep + { + public PublishStep():base("push", "dotnet","nuget", "push", "", "-s", "", "-k", ""){ } + + public override int Run(CIJob job) + { + bool success = true; + + CommandRunner.Arguments[4] = "http://nuget.l--n.de/nuget/l--n/v3/index.json"; + CommandRunner.Arguments[6] = "3yAJPMxcaEhb_HP62dxK"; + + foreach (string package in job.GetVariable("DOTNET_PACKAGES","").Split(':',StringSplitOptions.RemoveEmptyEntries)) + { + CommandRunner.Arguments[2] = package; + if (base.Run(job) != 0) + success = false; + } + return success ? 0 : -1; + } + } + + + } + +} \ No newline at end of file diff --git a/ln.build/PipeLine.cs b/ln.build/PipeLine.cs new file mode 100644 index 0000000..74624e5 --- /dev/null +++ b/ln.build/PipeLine.cs @@ -0,0 +1,73 @@ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json.Serialization; +using ln.logging; + +namespace ln.build +{ + + public class PipeLine + { + + List steps = new List(); + + public PipeLine() + { + } + + public void AddStep(Step step) => steps.Add(step); + + public virtual void Run(CIJob job) + { + job.Logger.Log(LogLevel.INFO,"PipeLine [{0}] start",GetType().Name); + + foreach (Step step in steps) + { + if (step.Run(job) != 0) + break; + } + + job.Logger.Log(LogLevel.INFO,"PipeLine [{0}] ended", GetType().Name); + } + + public abstract class Step + { + public string Name { get; } + public Step(string stepName) + { + Name = stepName; + } + + public abstract int Run(CIJob job); + + public string DefaultLogFileName => String.Format("step.{0}.log", Name); + } + + } + + public delegate void CommandExitedDelegate(CommandStep commandStep, CIJob job, int exitCode); + public class CommandStep : PipeLine.Step + { + public event CommandExitedDelegate OnCommandExited; + + public CommandRunner CommandRunner { get; } + + public CommandStep(string stepName, string filename,params string[] arguments) + :base(stepName) + { + CommandRunner = new CommandRunner(filename, arguments); + } + + public override int Run(CIJob job) + { + CommandRunner.WorkingDirectory = job.WorkingDirectory; + + int result = CommandRunner.Run(job.Logger, job.GetLogStream(DefaultLogFileName)); + OnCommandExited?.Invoke(this, job, result); + return result; + } + } + +} \ No newline at end of file diff --git a/ln.build/ln.build.csproj b/ln.build/ln.build.csproj new file mode 100644 index 0000000..2d2e5a2 --- /dev/null +++ b/ln.build/ln.build.csproj @@ -0,0 +1,23 @@ + + + + netcoreapp3.1 + + + + 0.1.0 + Harald Wolff-Thobaben + l--n.de + A simple build server scheduling builds triggered via web-hooks + (c) 2020 Harald Wolff-Thobaben + build build-server + + + + + + + + + +