Initial Commit
ln.build build job pending

master
Harald Wolff 2020-11-27 22:12:01 +01:00
commit 182d0be12a
10 changed files with 714 additions and 0 deletions

41
.gitignore vendored 100644
View File

@ -0,0 +1,41 @@
# Autosave files
*~
# build
[Oo]bj/
[Bb]in/
packages/
TestResults/
# globs
Makefile.in
*.DS_Store
*.sln.cache
*.suo
*.cache
*.pidb
*.userprefs
*.usertasks
config.log
config.make
config.status
aclocal.m4
install-sh
autom4te.cache/
*.user
*.tar.gz
tarballs/
test-results/
Thumbs.db
.vs/
# Mac bundle stuff
*.dmg
*.app
# resharper
*_Resharper.*
*.Resharper
# dotCover
*.dotCover

View File

@ -0,0 +1,93 @@
using System;
using System.Collections.Generic;
using ln.http;
using ln.http.router;
using ln.json;
using ln.logging;
using ln.type;
using ln.threading;
using ln.build;
namespace ln.build.server
{
class Program
{
static Pool pool = new Pool(2);
static HttpResponse WebHookRequest(HttpRoutingContext context, HttpRequest request)
{
HttpResponse response = new HttpResponse(request);
if (!request.Method.Equals("POST"))
{
response.StatusCode = 405;
} else if (!request.GetRequestHeader("content-type").Equals("application/json"))
{
response.StatusCode = 415;
response.ContentWriter.WriteLine("Unsupported Media Type, should be application/json");
} else {
JSONValue jsonRequest = JSONParser.Parse(request.ContentReader.ReadToEnd());
if (jsonRequest is JSONObject message)
{
try
{
string repoName = message["repository"]["name"].ToNative().ToString();
string cloneUrl = message["repository"]["clone_url"].ToNative().ToString();
string notifyEmail = message["pusher"]["email"].ToNative().ToString();
foreach (JSONValue commit in (message["commits"] as JSONArray).Children)
{
string commitID = commit["id"].ToNative().ToString();
Logging.Log("Received CI request for repository {2} [{0}] for the commit {1}", cloneUrl, commitID, repoName);
CIJob job = new CIJob(repoName, cloneUrl, commitID, notifyEmail);
job.SetVariable("REPO_OWNER", message["repository"]["owner"]["username"].ToNative().ToString());
job.SetVariable("REPO_NAME", message["repository"]["name"].ToNative().ToString());
job.SetVariable("COMMIT_ID", commitID);
job.UpdateBuildState(BuildState.PENDING);
pool.Enqueue(job);
}
} catch (Exception e)
{
response.StatusCode = 500;
response.StatusMessage = "An exception occured";
response.ContentWriter.WriteLine("{0}", e.ToString());
Logging.Log(e);
}
} else {
response.StatusCode = 400;
}
}
return response;
}
static void Main(string[] args)
{
CommandRunner.DefaultThrow = CRThrow.NEGATIVE;
SimpleRouter genericRouter = new SimpleRouter();
genericRouter.AddSimpleRoute("/", WebHookRequest);
HTTPServer httpServer = new HTTPServer(new Endpoint(IPv6.ANY, 1888), new LoggingRouter(genericRouter));
pool.Start();
Logging.Log("Starting http listener...");
httpServer.Start();
}
}
}

View File

@ -0,0 +1,29 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
<PropertyGroup>
<Version>0.1.0</Version>
<Authors>Harald Wolff-Thobaben</Authors>
<Company>l--n.de</Company>
<Description>A simple build server scheduling builds triggered via web-hooks</Description>
<Copyright>(c) 2020 Harald Wolff-Thobaben</Copyright>
<PackageTags>build build-server</PackageTags>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="ln.application" Version="0.1.1" />
<PackageReference Include="ln.http" Version="0.1.1" />
<PackageReference Include="ln.json" Version="1.0.0" />
<PackageReference Include="ln.logging" Version="1.0.1" />
<PackageReference Include="ln.threading" Version="0.1.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../ln.build/ln.build.csproj" />
</ItemGroup>
</Project>

48
ln.build.sln 100644
View File

@ -0,0 +1,48 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.26124.0
MinimumVisualStudioVersion = 15.0.26124.0
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ln.build", "ln.build\ln.build.csproj", "{682A1FD5-3722-4C16-BED9-FFD90ED40FE8}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ln.build.server", "ln.build.server\ln.build.server.csproj", "{3437B6AB-7937-42DD-A526-5716E0114C61}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Debug|x64 = Debug|x64
Debug|x86 = Debug|x86
Release|Any CPU = Release|Any CPU
Release|x64 = Release|x64
Release|x86 = Release|x86
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Debug|x64.ActiveCfg = Debug|Any CPU
{682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Debug|x64.Build.0 = Debug|Any CPU
{682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Debug|x86.ActiveCfg = Debug|Any CPU
{682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Debug|x86.Build.0 = Debug|Any CPU
{682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Release|Any CPU.ActiveCfg = Release|Any CPU
{682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Release|Any CPU.Build.0 = Release|Any CPU
{682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Release|x64.ActiveCfg = Release|Any CPU
{682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Release|x64.Build.0 = Release|Any CPU
{682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Release|x86.ActiveCfg = Release|Any CPU
{682A1FD5-3722-4C16-BED9-FFD90ED40FE8}.Release|x86.Build.0 = Release|Any CPU
{3437B6AB-7937-42DD-A526-5716E0114C61}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3437B6AB-7937-42DD-A526-5716E0114C61}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3437B6AB-7937-42DD-A526-5716E0114C61}.Debug|x64.ActiveCfg = Debug|Any CPU
{3437B6AB-7937-42DD-A526-5716E0114C61}.Debug|x64.Build.0 = Debug|Any CPU
{3437B6AB-7937-42DD-A526-5716E0114C61}.Debug|x86.ActiveCfg = Debug|Any CPU
{3437B6AB-7937-42DD-A526-5716E0114C61}.Debug|x86.Build.0 = Debug|Any CPU
{3437B6AB-7937-42DD-A526-5716E0114C61}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3437B6AB-7937-42DD-A526-5716E0114C61}.Release|Any CPU.Build.0 = Release|Any CPU
{3437B6AB-7937-42DD-A526-5716E0114C61}.Release|x64.ActiveCfg = Release|Any CPU
{3437B6AB-7937-42DD-A526-5716E0114C61}.Release|x64.Build.0 = Release|Any CPU
{3437B6AB-7937-42DD-A526-5716E0114C61}.Release|x86.ActiveCfg = Release|Any CPU
{3437B6AB-7937-42DD-A526-5716E0114C61}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
EndGlobal

View File

@ -0,0 +1,6 @@
namespace ln.build
{
public enum BuildState { PENDING, SUCCESS, ERROR, FAILURE, WARNING }
}

175
ln.build/CIJob.cs 100644
View File

@ -0,0 +1,175 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Reflection;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using ln.json;
using ln.logging;
using ln.threading;
namespace ln.build
{
public class CIJob : PoolJob
{
static HttpClient httpClient = new HttpClient();
public string JobID { get; } = Guid.NewGuid().ToString("N");
public string RepositoryURL { get; }
public string RepositoryName { get; }
public string Commit { get; }
public string NotifyEMail { get; set; }
public string WorkingDirectory { get; set; }
public Logger Logger { get; private set; }
List<PipeLine> pipeLines = new List<PipeLine>();
Dictionary<string,string> variables = new Dictionary<string, string>();
Dictionary<string,MemoryStream> logStreams = new Dictionary<string, MemoryStream>();
Dictionary<string,Logger> logStreamLoggers = new Dictionary<string, Logger>();
public CIJob(string repositoryName, string repositoryURL, string commit, string notifyEMail)
{
WorkingDirectory = Path.Combine(Path.GetTempPath(), JobID);
Logger = new Logger(new FileLogger(Path.Combine(Path.GetTempPath(), String.Format("{0}.log", JobID))));
Logger.Backends.Add(Logger.ConsoleLogger);
RepositoryName = repositoryName;
RepositoryURL = repositoryURL;
Commit = commit;
NotifyEMail = notifyEMail;
}
public Stream GetLogStream(string name)
{
if (!logStreams.TryGetValue(name,out MemoryStream logStream))
{
logStream = new MemoryStream();
logStreams.Add(name, logStream);
}
return logStream;
}
public Logger GetLogger(string name)
{
if (!logStreamLoggers.TryGetValue(name, out Logger logStreamLogger))
{
logStreamLogger = new Logger(GetLogStream(name));
logStreamLogger.Backends.Add(Logger.ConsoleLogger);
logStreamLoggers.Add(name, logStreamLogger);
}
return logStreamLogger;
}
public string GetVariable(string varName) => GetVariable(varName, null);
public string GetVariable(string varName,string defValue)
{
if (!variables.TryGetValue(varName,out string value))
value = defValue;
return value;
}
public void SetVariable(string varName,string value)
{
if (value != null)
variables[varName] = value;
else
variables.Remove(varName);
}
public void ExtendVariable(string varName,string value) => ExtendVariable(varName, value, ':');
public void ExtendVariable(string varName, string value, char seperator)
{
String currentValue = GetVariable(varName, "");
if (String.Empty.Equals(currentValue))
{
currentValue = value;
} else {
currentValue = string.Format("{0}{1}{2}",currentValue, seperator, value);
}
SetVariable(varName, currentValue);
}
public async void UpdateBuildState(BuildState buildState)
{
string buildStateURL = String.Format("https://git.l--n.de/api/v1/repos/{0}/{1}/statuses/{2}",
GetVariable("REPO_OWNER"),
GetVariable("REPO_NAME"),
GetVariable("COMMIT_ID")
);
if (buildStateURL != null)
{
JSONObject stateObject = new JSONObject();
stateObject.Add("context", "ln.build");
stateObject.Add("description", "build job pending");
stateObject.Add("state", buildState.ToString().ToLower());
stateObject.Add("target_url", JSONNull.Instance);
HttpResponseMessage response = await httpClient.PostAsync(buildStateURL, new StringContent(stateObject.ToString(),Encoding.UTF8,"application/json"));
Logger.Log(LogLevel.DEBUG, "UpdateBuildState({0}): {1}", buildState, buildStateURL);
Logger.Log(LogLevel.DEBUG, "Response: {0}", response );
}
}
public override void RunJob()
{
CloneRepository();
DetectPipelines();
try{
foreach (PipeLine pipeLine in pipeLines)
{
pipeLine.Run(this);
}
UpdateBuildState(BuildState.SUCCESS);
} catch (Exception e)
{
UpdateBuildState(BuildState.FAILURE);
}
Notify();
Cleanup();
}
void DetectPipelines()
{
string[] sln = Directory.GetFileSystemEntries(WorkingDirectory,"*.sln");
if (sln.Length > 0)
{
pipeLines.Add(new DotNetPipeLine());
}
}
public void CloneRepository()
{
Logging.Log("cloning repository to {0}", WorkingDirectory);
new CommandRunner(Logger, "git", "clone",RepositoryURL,WorkingDirectory).Run();
new CommandRunner(Logger, "git", "checkout", Commit).Run();
}
public void Notify()
{
}
public void Cleanup()
{
Directory.Delete(WorkingDirectory, true);
}
static CIJob()
{
httpClient.DefaultRequestHeaders.Add("Authorization","token 1d03e9577c404b5b4f46b340147b1d500ff95b2e");
}
}
}

View File

@ -0,0 +1,127 @@
using System;
using System.Diagnostics;
using System.IO;
using ln.logging;
namespace ln.build
{
public enum CRThrow { NEVER, NEGATIVE, NONZERO }
public class CommandRunner
{
public static CRThrow DefaultThrow { get; set; } = CRThrow.NONZERO;
public CRThrow Throw { get; set; } = DefaultThrow;
public string Executable { get; }
public string[] Arguments { get; set; }
public string WorkingDirectory { get; set; } = ".";
public Logger Logger { get; }
public CommandRunner(string executable,params string[] arguments) : this(Logger.Default, executable, arguments){}
public CommandRunner(Logger logger, string executable,params string[] arguments)
{
Logger = logger;
Executable = executable;
Arguments = arguments;
}
public string FindFileInPath(string filename)
{
Logger.Log(LogLevel.DEBUG, "Looking up {0} in paths {1}", filename, Environment.GetEnvironmentVariable("PATH"));
foreach (string path in Environment.GetEnvironmentVariable("PATH").Split(Path.PathSeparator))
{
string fullpath = Path.Combine(path,filename);
if (File.Exists(fullpath))
return fullpath;
}
return filename;
}
public int Run() => Run(Logger, null, null);
public int Run(Stream stdout, Stream stderr) => Run(Logger, stdout, stderr);
public int Run(Logger logger, Stream stdout) => Run(logger, stdout, stdout);
public int Run(Logger logger, Stream stdout, Stream stderr)
{
ProcessStartInfo psi = new ProcessStartInfo(FindFileInPath(Executable), string.Join(' ', Arguments))
{
CreateNoWindow = true,
RedirectStandardError = true,
RedirectStandardOutput = true,
RedirectStandardInput = true,
WorkingDirectory = this.WorkingDirectory
};
psi.EnvironmentVariables.Remove("LANG");
logger.Log(LogLevel.INFO, "Executing: {0} {1}", psi.FileName, psi.Arguments);
Process process = Process.Start(psi);
process.WaitForExit();
if (stdout != null)
process.StandardOutput.BaseStream.CopyTo(stdout);
if (stderr != null)
process.StandardError.BaseStream.CopyTo(stderr);
logger.Log(LogLevel.INFO, "Result: {0}", process.ExitCode);
if (
((Throw == CRThrow.NEGATIVE) && (process.ExitCode < 0)) ||
((Throw == CRThrow.NONZERO) && (process.ExitCode != 0))
)
throw new Exception(String.Format("{0} execution gave result {1}", psi.FileName, process.ExitCode));
return process.ExitCode;
}
public int Run(out string stdout,out string stderr) => Run(Logger, out stdout, out stderr);
public int Run(Logger logger, out string stdout,out string stderr)
{
MemoryStream outstream,errstream;
outstream = new MemoryStream();
errstream = new MemoryStream();
int status = Run(logger, outstream, errstream);
using (StreamReader sr = new StreamReader(outstream))
stdout = sr.ReadToEnd();
using (StreamReader sr = new StreamReader(errstream))
stderr = sr.ReadToEnd();
return status;
}
/*
public static int Run(string executable,out Stream stdout,out Stream stderr,params string[] arguments)
{
CommandRunner runner = new CommandRunner(executable,arguments);
return runner.Run(out stdout,out stderr);
}
public static int Run(string executable,out string stdout,out string stderr,params string[] arguments) => Run(Logger.Default, executable, out stdout, out stderr, arguments);
public static int Run(Logger logger, string executable, out string stdout, out string stderr, params string[] arguments)
{
CommandRunner runner = new CommandRunner(logger, executable, arguments);
int result = runner.Run(out stdout, out stderr);
return result;
}
public static int Run(string executable,params string[] arguments) => Run(executable,out Stream stdout,out Stream stderr,arguments);
public static int Run(Logger logger, string executable, params string[] arguments){
return Run(executable,out Stream stdout,out Stream stderr,arguments);
}
*/
}
}

View File

@ -0,0 +1,99 @@
using System;
using System.IO;
using System.Reflection.Metadata.Ecma335;
using System.Runtime.Serialization.Formatters;
using System.Text;
using System.Text.RegularExpressions;
using LibGit2Sharp.Handlers;
using ln.logging;
using ln.type;
namespace ln.build
{
public class DotNetPipeLine : PipeLine
{
static Regex regexPackages = new Regex(".*(Successfully created package '(.*)').*");
public DotNetPipeLine()
{
AddStep(new RestoreStep());
AddStep(new CleanStep());
BuildStep buildStep = new BuildStep();
buildStep.OnCommandExited += FilterPackageFilenames;
AddStep(buildStep);
AddStep(new TestStep());
PackStep packStep = new PackStep();
packStep.OnCommandExited += FilterPackageFilenames;
AddStep(packStep);
AddStep(new PublishStep());
}
void FilterPackageFilenames(CommandStep commandStep, CIJob job, int exitCode)
{
Stream outStream = job.GetLogStream(commandStep.DefaultLogFileName);
outStream.Position = 0;
job.Logger.Log(LogLevel.INFO,"filterPackageFilenames(): started");
String consoleOutput = Encoding.UTF8.GetString(outStream.ReadToEnd());
MatchCollection matches = regexPackages.Matches(consoleOutput);
foreach (Match match in matches)
{
Logging.Log(LogLevel.INFO,"filterPackageFilenames(): {0}", match.Groups[2].Value);
job.ExtendVariable("DOTNET_PACKAGES", match.Groups[2].Value);
}
}
class CleanStep : CommandStep
{
public CleanStep():base("clean", "dotnet","clean"){ }
}
class RestoreStep : CommandStep
{
public RestoreStep():base("restore", "dotnet","restore"){ }
}
class TestStep : CommandStep
{
public TestStep():base("test", "dotnet","test"){ }
}
class BuildStep : CommandStep
{
public BuildStep():base("build", "dotnet","build"){ }
}
class PackStep : CommandStep
{
public PackStep():base("pack", "dotnet","pack"){ }
}
class PublishStep : CommandStep
{
public PublishStep():base("push", "dotnet","nuget", "push", "<filename>", "-s", "<source>", "-k", "<apikey>"){ }
public override int Run(CIJob job)
{
bool success = true;
CommandRunner.Arguments[4] = "http://nuget.l--n.de/nuget/l--n/v3/index.json";
CommandRunner.Arguments[6] = "3yAJPMxcaEhb_HP62dxK";
foreach (string package in job.GetVariable("DOTNET_PACKAGES","").Split(':',StringSplitOptions.RemoveEmptyEntries))
{
CommandRunner.Arguments[2] = package;
if (base.Run(job) != 0)
success = false;
}
return success ? 0 : -1;
}
}
}
}

View File

@ -0,0 +1,73 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text.Json.Serialization;
using ln.logging;
namespace ln.build
{
public class PipeLine
{
List<Step> steps = new List<Step>();
public PipeLine()
{
}
public void AddStep(Step step) => steps.Add(step);
public virtual void Run(CIJob job)
{
job.Logger.Log(LogLevel.INFO,"PipeLine [{0}] start",GetType().Name);
foreach (Step step in steps)
{
if (step.Run(job) != 0)
break;
}
job.Logger.Log(LogLevel.INFO,"PipeLine [{0}] ended", GetType().Name);
}
public abstract class Step
{
public string Name { get; }
public Step(string stepName)
{
Name = stepName;
}
public abstract int Run(CIJob job);
public string DefaultLogFileName => String.Format("step.{0}.log", Name);
}
}
public delegate void CommandExitedDelegate(CommandStep commandStep, CIJob job, int exitCode);
public class CommandStep : PipeLine.Step
{
public event CommandExitedDelegate OnCommandExited;
public CommandRunner CommandRunner { get; }
public CommandStep(string stepName, string filename,params string[] arguments)
:base(stepName)
{
CommandRunner = new CommandRunner(filename, arguments);
}
public override int Run(CIJob job)
{
CommandRunner.WorkingDirectory = job.WorkingDirectory;
int result = CommandRunner.Run(job.Logger, job.GetLogStream(DefaultLogFileName));
OnCommandExited?.Invoke(this, job, result);
return result;
}
}
}

View File

@ -0,0 +1,23 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
<PropertyGroup>
<Version>0.1.0</Version>
<Authors>Harald Wolff-Thobaben</Authors>
<Company>l--n.de</Company>
<Description>A simple build server scheduling builds triggered via web-hooks</Description>
<Copyright>(c) 2020 Harald Wolff-Thobaben</Copyright>
<PackageTags>build build-server</PackageTags>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="LibGit2Sharp" Version="0.26.2" />
<PackageReference Include="ln.logging" Version="1.0.1" />
<PackageReference Include="ln.threading" Version="0.1.0" />
<PackageReference Include="ln.json" Version="1.0.0" />
</ItemGroup>
</Project>