ln-dotnet
/
ln.build
3
0
Fork 0
Code Issues 2 Pull Requests Releases 5 Wiki Activity

Beta 0.2.0

master
Harald Wolff 2020-12-03 00:05:51 +01:00
parent 207b593e7d
commit baf45f4e00
15 changed files with 356 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -39,3 +39,8 @@ Thumbs.db
# dotCover
*.dotCover
*.log
*.log.old
.vscode
.build

14
build.ln
View File

@ -25,21 +25,25 @@
{
"name": "build",
"commands": [
"SH dotnet build -c Release"
"SH dotnet build -c $CONFIGURATION"
]
},
{
"name": "pack_and_publish",
"commands": [
"SH dotnet pack ln.build -o .build -c Release",
"SH dotnet publish ln.build.server -p:PublishTrimmed=true -p:PublishSingleFile=true -p:PublishReadyToRun=true --self-contained -r linux-x64 -c Release -o .build/linux-x64"
"SH dotnet pack ln.build -o .build -c $CONFIGURATION",
"SH dotnet pack ln.build.server -o .build -c $CONFIGURATION",
"SH dotnet publish ln.build.server -p:PublishTrimmed=true -p:PublishSingleFile=true -p:PublishReadyToRun=true --self-contained -r linux-x64 -c $CONFIGURATION -o .build/linux-x64"
]
},
{
"name": "push",
"commands": [
"SH dotnet nuget push .build/ln-build.*.nupkg -s ${NUGET_SOURCE} -k ${NUGET_APIKEY}"
]
"SH dotnet nuget push .build/ln.build.*.nupkg -s $NUGET_SOURCE -k $NUGET_APIKEY"
],
"secrets": {
"NUGET_APIKEY": "key/nuget.l--n.de"
}
}
]
}

18
ln.build.server/Program.cs
View File

@ -18,6 +18,9 @@ namespace ln.build.server
{
static CIService CIService;
[StaticArgument( LongOption = "build")]
public static string BuildPath { get; set; }
static void Main(string[] args)
{
ArgumentContainer ac = new ArgumentContainer(typeof(Program));
@ -27,10 +30,19 @@ namespace ln.build.server
ac.Parse(ref args);
CIService.Initialize();
CIService.AddPipeline(new DotNetPipeLine());
CIService.AddRepositoryInterface(new GiteaRepositoryInterface("https://git.l--n.de"){ AuthorizationToken = "1d03e9577c404b5b4f46b340147b1d500ff95b2e", });
CIService.Start();
if (BuildPath != null)
{
CIJob job = new CIJob(CIService,null);
job.WorkingDirectory = BuildPath;
job.RunJob();
} else {
CIService.Initialize();
CIService.AddRepositoryInterface(new GiteaRepositoryInterface("https://git.l--n.de"){ AuthorizationToken = "1d03e9577c404b5b4f46b340147b1d500ff95b2e", });
CIService.Start();
}
}

2
ln.build.server/ln.build.server.csproj
View File

@ -6,7 +6,7 @@
</PropertyGroup>
<PropertyGroup>
<Version>0.1.0-test1</Version>
<Version>0.2.0</Version>
<Authors>Harald Wolff-Thobaben</Authors>
<Company>l--n.de</Company>
<Description>A simple build server scheduling builds triggered via web-hooks</Description>

53
ln.build/CIJob.cs
View File

@ -4,6 +4,7 @@ using System.IO;
using System.Net.Http;
using System.Text;
using ln.build.commands;
using ln.build.pipeline;
using ln.build.repositories;
using ln.json;
using ln.logging;
@ -24,7 +25,7 @@ namespace ln.build
public RepositoryInterface RepositoryInterface { get; set; }
List<PipeLine> pipeLines = new List<PipeLine>();
public DefaultPipeLine PipeLine { get; set; }
public BuildState BuildState { get; private set; }
@ -39,7 +40,9 @@ namespace ln.build
RepositoryInterface = repositoryInterface;
WorkingDirectory = Path.Combine(Path.GetTempPath(), JobID);
Logger = new Logger(new FileLogger(Path.Combine(Path.GetTempPath(), String.Format("{0}.log", JobID))));
Directory.CreateDirectory(Path.Combine(ciService.ReportsDirectory, JobID));
Logger = new Logger(new FileLogger(Path.Combine(ciService.ReportsDirectory, JobID, "build.log")));
Logger.Backends.Add(Logger.ConsoleLogger);
RepositoryURL = repositoryURL;
@ -90,20 +93,17 @@ namespace ln.build
{
if (CloneRepository())
{
if (DetectPipelines())
try
{
UpdateBuildState(BuildState.PENDING);
LoadPipeLine();
try{
foreach (PipeLine pipeLine in pipeLines)
{
pipeLine.Run(this);
}
UpdateBuildState(BuildState.SUCCESS);
} catch (Exception e)
{
UpdateBuildState(BuildState.FAILURE);
}
PipeLine.Run();
UpdateBuildState(BuildState.SUCCESS);
} catch (Exception e)
{
Logger.Log(e);
UpdateBuildState(BuildState.FAILURE);
}
} else {
Logger.Log(LogLevel.ERROR, "CIJob failed at CloneRepository()");
@ -115,23 +115,20 @@ namespace ln.build
public bool CloneRepository()
{
Logging.Log("cloning repository to {0}", WorkingDirectory);
return
(new CommandRunner(Logger, "git", "clone", RepositoryURL, WorkingDirectory).Run(Environment) == 0) &&
(!ContainsVariable("COMMIT_ID") || new CommandRunner(Logger, "git", "checkout", GetVariable("COMMIT_ID")){ WorkingDirectory = this.WorkingDirectory, }.Run(Environment) == 0);
RepositoryInterface?.CloneSources(this);
return true;
}
public bool DetectPipelines()
public void LoadPipeLine()
{
if (CIService != null)
JSONObject jsonPipeLine = JSONParser.ParseFile(Path.Combine(WorkingDirectory, "build.ln")) as JSONObject;
if (jsonPipeLine != null)
{
foreach (PipeLine pipeLine in CIService.PipeLines)
{
if (pipeLine.DetectValidity(this))
pipeLines.Add(pipeLine);
}
DefaultPipeLine pipeLine = new DefaultPipeLine(CIService, Environment);
pipeLine.LoadJson(jsonPipeLine);
PipeLine = pipeLine;
}
return pipeLines.Count > 0;
}
public void Notify()
@ -150,7 +147,9 @@ namespace ln.build
public void Cleanup()
{
CIService.CreateReport(this);
Directory.Delete(WorkingDirectory, true);
if (RepositoryInterface != null)
Directory.Delete(WorkingDirectory, true);
}
}

18
ln.build/CIService.cs
View File

@ -44,9 +44,26 @@ namespace ln.build
HashSet<PipeLine> pipelines = new HashSet<PipeLine>();
public IEnumerable<PipeLine> PipeLines => pipelines;
Dictionary<string,string> secrets = new Dictionary<string, string>();
public CIService()
{
buildPool = new Pool(2);
String secretsPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),".ln.build.secrets.json");
Logging.Log(LogLevel.INFO, "Trying to load secrets from {0}", secretsPath);
if (File.Exists(secretsPath))
{
JSONObject secretsObject = JSONParser.ParseFile(secretsPath) as JSONObject;
foreach (string key in secretsObject.Keys)
{
secrets.Add(key, secretsObject[key].ToNative().ToString());
}
Logging.Log(LogLevel.INFO, "Secrets loaded from {0}", secretsPath);
}
}
public CIService(string baseDirectory) : this()
{
@ -113,6 +130,7 @@ namespace ln.build
public string GetJobURL(CIJob job) => string.Format("{0}/builds/{1}", BaseURL, job.JobID);
public string GetSecret(string key) => secrets[key];
public void Start()
{

42
ln.build/commands/CommandEnvironment.cs
View File

@ -3,12 +3,19 @@ using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using ln.json;
using ln.logging;
namespace ln.build.commands
{
public class CommandEnvironment : IDictionary<string,string>
{
public CommandEnvironment Parent { get; }
public string WorkingDirectory { get; set; } = Path.GetFullPath(".");
static string[] DefaultVariables = { "PATH", "HOME", "USERNAME" };
public Logger Logger { get; set; }
Dictionary<string,string> variables = new Dictionary<string, string>();
@ -19,14 +26,16 @@ namespace ln.build.commands
public bool IsReadOnly => ((ICollection<KeyValuePair<string, string>>)variables).IsReadOnly;
public string this[string key] { get => Get(key); set => Set(key,value); }
public CommandEnvironment(CommandEnvironment parent) : this(parent.Logger) { Parent = parent; WorkingDirectory = parent.WorkingDirectory; }
public CommandEnvironment(CommandEnvironment parent, Logger logger) : this(logger) { Parent = parent; WorkingDirectory = parent.WorkingDirectory; }
public CommandEnvironment() : this(Logger.Default)
{
foreach (string defName in DefaultVariables)
Set(defName, Environment.GetEnvironmentVariable(defName));
}
{ }
public CommandEnvironment(Logger logger)
{
Logger = logger;
foreach (string defName in DefaultVariables)
Set(defName, Environment.GetEnvironmentVariable(defName));
}
public CommandEnvironment(IEnumerable<KeyValuePair<string,string>> setup) : this(Logger.Default, setup) { }
public CommandEnvironment(Logger logger, IEnumerable<KeyValuePair<string,string>> setup) : this(logger)
@ -40,7 +49,12 @@ namespace ln.build.commands
public string Get(string varName,string defValue)
{
if (!variables.TryGetValue(varName,out string value))
value = defValue;
{
if (Parent != null)
value = Parent.Get(varName, defValue);
else
value = defValue;
}
return value;
}
public CommandEnvironment Set(string varName,string value)
@ -123,9 +137,25 @@ namespace ln.build.commands
public IEnumerator<KeyValuePair<string, string>> GetEnumerator()
{
return ((IEnumerable<KeyValuePair<string, string>>)variables).GetEnumerator();
if (Parent != null)
{
return Keys.Concat(Parent.Keys).Distinct().Select((k)=>new KeyValuePair<string, string>(k,Get(k))).GetEnumerator();
} else
{
return ((IEnumerable<KeyValuePair<string, string>>)variables).GetEnumerator();
}
}
public void Apply(JSONObject jsonEnvironment)
{
foreach (string key in jsonEnvironment?.Keys ?? new string[0])
{
Set(key, jsonEnvironment[key].ToNative().ToString());
}
}
IEnumerator IEnumerable.GetEnumerator()
{
return ((IEnumerable)variables).GetEnumerator();

4
ln.build/commands/CommandRunner.cs
View File

@ -18,8 +18,6 @@ namespace ln.build.commands
List<Argument> arguments = new List<Argument>();
public IEnumerable<Argument> Arguments => arguments;
public string WorkingDirectory { get; set; } = ".";
public Logger Logger { get; }
Func<int,bool> TestExitCode = null;
@ -67,7 +65,7 @@ namespace ln.build.commands
RedirectStandardError = true,
RedirectStandardOutput = true,
RedirectStandardInput = true,
WorkingDirectory = this.WorkingDirectory,
WorkingDirectory = environment.WorkingDirectory
};
psi.Environment.Clear();

2
ln.build/ln.build.csproj
View File

@ -5,7 +5,7 @@
</PropertyGroup>
<PropertyGroup>
<Version>0.1.0-test2</Version>
<Version>0.2.0</Version>
<Authors>Harald Wolff-Thobaben</Authors>
<Company>l--n.de</Company>
<Description>A simple build server scheduling builds triggered via web-hooks</Description>

184
ln.build/pipeline/DefaultPipeLine.cs 100644
View File

@ -0,0 +1,184 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using ln.build.commands;
using ln.json;
using ln.logging;
namespace ln.build.pipeline
{
public class DefaultPipeLine
{
public CommandEnvironment CommandEnvironment { get; }
public CIService CIService { get; }
List<Stage> stages = new List<Stage>();
public IEnumerable<Stage> Stages => stages;
public DefaultPipeLine(CIService ciService)
{
CIService = ciService;
CommandEnvironment = new CommandEnvironment();
}
public DefaultPipeLine(CIService ciService, CommandEnvironment commandEnvironment)
{
CIService = ciService;
CommandEnvironment = new CommandEnvironment(commandEnvironment);
}
public void LoadJson(JSONObject jsonPipeLine)
{
if (jsonPipeLine.ContainsKey("env"))
CommandEnvironment.Apply(jsonPipeLine["env"] as JSONObject);
if (jsonPipeLine.ContainsKey("stages"))
{
JSONArray jsonStages = jsonPipeLine["stages"] as JSONArray;
foreach (JSONObject jsonStage in jsonStages.Children)
{
Stage stage = new Stage(this);
stage.LoadJson(jsonStage);
stages.Add(stage);
}
}
}
public void Run()
{
foreach (Stage stage in stages)
{
CommandEnvironment.Logger.Log(LogLevel.INFO,"-------------------------------------------------------------------------------------");
CommandEnvironment.Logger.Log(LogLevel.INFO,"STAGE: {0}", stage.Name);
CommandEnvironment.Logger.Log(LogLevel.INFO,"-------------------------------------------------------------------------------------");
stage.Run();
}
}
}
public class Stage
{
public DefaultPipeLine PipeLine { get; }
public string Name { get; set; }
public CommandEnvironment CommandEnvironment { get; }
List<StageCommand> commands = new List<StageCommand>();
public IEnumerable<StageCommand> Commands => commands;
public Stage(DefaultPipeLine pipeLine)
{
PipeLine = pipeLine;
CommandEnvironment = new CommandEnvironment(pipeLine.CommandEnvironment);
}
public void LoadJson(JSONObject jsonStage)
{
Name = jsonStage["name"].ToNative().ToString();
if (jsonStage.ContainsKey("env"))
{
CommandEnvironment.Apply(jsonStage["env"] as JSONObject);
}
if (jsonStage.ContainsKey("commands"))
{
JSONArray jsonCommands = jsonStage["commands"] as JSONArray;
foreach (JSONValue jsonValue in jsonCommands.Children)
{
commands.Add(StageCommand.Create(jsonValue.ToNative().ToString()));
}
}
if (jsonStage.ContainsKey("secrets"))
{
JSONObject jsonSecrets = jsonStage["secrets"] as JSONObject;
foreach (string key in jsonSecrets.Keys)
{
CommandEnvironment.Set(key, PipeLine.CIService.GetSecret(jsonSecrets[key].ToNative().ToString()));
}
}
}
public void Run()
{
foreach (StageCommand command in commands)
command.Run(this);
}
}
public abstract class StageCommand
{
public string Name { get; }
public StageCommand(string name)
{
Name = name;
}
public abstract void Run(Stage stage);
static Dictionary<string,Func<string,StageCommand>> commandFactories = new Dictionary<string, Func<string, StageCommand>>();
public static StageCommand Create(string cmdline)
{
string[] tokens = cmdline.Split(new char[]{' ','\t'}, 2);
if (commandFactories.TryGetValue(tokens[0],out Func<string,StageCommand> factory))
{
return factory(tokens[1]);
}
throw new Exception(string.Format("can't find factory for command keyword '{0}'", tokens[0]));
}
static StageCommand()
{
commandFactories.Add("SH", (args) => new ShellCommand(args));
commandFactories.Add("DOTNET", (args) => new DotNetCommand(args));
commandFactories.Add("GITEA", (args) => new DotNetCommand(args));
}
}
public class ShellCommand : StageCommand
{
public event CommandExitedDelegate OnCommandExited;
public CommandRunner CommandRunner { get; }
public ShellCommand(string filename,params CommandRunner.Argument[] arguments)
:base("SH")
{
CommandRunner = new CommandRunner("/bin/bash", "-c"){ Throw = CRThrow.NEVER, };
CommandRunner.AddArguments(arguments);
}
public ShellCommand(string cmdline)
:base("SH")
{
CommandRunner = new CommandRunner("/bin/bash", "-c", string.Format("\"{0}\"",cmdline)){ Throw = CRThrow.NEVER, };
}
public override void Run(Stage stage)
{
MemoryStream logStream = new MemoryStream();
int result = CommandRunner.Run(stage.CommandEnvironment, logStream);
stage.CommandEnvironment.Logger.Log(LogLevel.INFO, "command output:\n{0}", Encoding.UTF8.GetString(logStream.ToArray()));
stage.CommandEnvironment.Logger.Log(LogLevel.INFO, "command exit code: {0}", result);
if (result != 0)
throw new Exception(String.Format("command exited with code {0} [0x{0:x}]", result));
}
}
}

26
ln.build/pipeline/DotNetCommand.cs 100644
View File

@ -0,0 +1,26 @@
using ln.build.pipeline;
namespace ln.build.commands
{
public class DotNetCommand : StageCommand
{
public DotNetCommand(string arguments) :base("DOTNET"){
}
public void Analyze(Stage stage)
{
}
public override void Run(Stage stage)
{
}
}
}

1
ln.build/DotNetPipeLine.cs → ln.build/pipeline/DotNetPipeLine.cs
View File

@ -114,7 +114,6 @@ namespace ln.build
return success ? 0 : -1;
}
}
}

4
ln.build/PipeLine.cs → ln.build/pipeline/PipeLine.cs
View File

@ -63,9 +63,7 @@ namespace ln.build
}
public override int Run(CIJob job)
{
CommandRunner.WorkingDirectory = job.WorkingDirectory;
{
int result = CommandRunner.Run(job.Environment, job.GetLogStream(DefaultLogFileName));
OnCommandExited?.Invoke(this, job, result);
return result;

12
ln.build/repositories/GitRepositoryInterface.cs
View File

@ -1,5 +1,6 @@
using System;
using System.IO;
using ln.build.commands;
using ln.http;
@ -18,8 +19,15 @@ namespace ln.build.repositories
public override void CloneSources(CIJob job)
{
job.Logger.Log("{0}: cloning repository to {1}", GetType().Name, job.WorkingDirectory);
bool success = (new CommandRunner("git", "clone", job.RepositoryURL, job.WorkingDirectory).Run(job.Environment) == 0) &&
(!job.ContainsVariable("COMMIT_ID") || new CommandRunner("git", "checkout", job.GetVariable("COMMIT_ID")){ WorkingDirectory = job.WorkingDirectory, }.Run(job.Environment) == 0);
job.Environment.WorkingDirectory = Path.GetTempPath();
bool success = new CommandRunner("git", "clone", job.RepositoryURL, job.WorkingDirectory).Run(job.Environment) == 0;
job.Environment.WorkingDirectory = job.WorkingDirectory;
if (success && job.ContainsVariable("COMMIT_ID"))
{
success = new CommandRunner("git", "checkout", job.GetVariable("COMMIT_ID")).Run(job.Environment) == 0;
}
if (!success)
throw new Exception("clone failed");
}

29
ln.build/repositories/GiteaRepositoryInterface.cs
View File

@ -19,6 +19,13 @@ namespace ln.build.repositories
BaseURL = baseURL;
}
HttpClient CreateHttpClient()
{
HttpClient httpClient = new HttpClient();
httpClient.DefaultRequestHeaders.Add("Authorization",String.Format("token {0}", AuthorizationToken));
return httpClient;
}
public override bool DetectValidity(string cloneUrl) => cloneUrl.StartsWith(BaseURL);
public override void UpdateBuildState(CIJob job)
{
@ -37,10 +44,8 @@ namespace ln.build.repositories
stateObject.Add("state", job.BuildState.ToString().ToLower());
stateObject.Add("target_url", job.CIService.GetJobURL(job));
using (HttpClient httpClient = new HttpClient())
using (HttpClient httpClient = CreateHttpClient())
{
httpClient.DefaultRequestHeaders.Add("Authorization",String.Format("token {0}", AuthorizationToken));
HttpResponseMessage response = httpClient.PostAsync(buildStateURL, new StringContent(stateObject.ToString(),Encoding.UTF8,"application/json")).Result;
job.Logger.Log(LogLevel.DEBUG, "UpdateBuildState({0}): {1}", job.BuildState, buildStateURL);
job.Logger.Log(LogLevel.DEBUG, "Request: {0}", stateObject.ToString());
@ -82,11 +87,23 @@ namespace ln.build.repositories
.SetVariable("NUGET_SOURCE", "http://nuget.l--n.de/nuget/l--n/v3/index.json")
;
job.UpdateBuildState(BuildState.PENDING);
using (HttpClient httpClient = CreateHttpClient())
{
string triggerFile = string.Format("{3}/api/v1/repos/{0}/{1}/contents/build.ln?ref={2}",
job.GetVariable("REPO_OWNER"),
job.GetVariable("REPO_NAME"),
job.GetVariable("COMMIT_ID"),
BaseURL
);
ciService.Enqueue(job);
HttpResponseMessage triggerResponse = httpClient.GetAsync(triggerFile).Result;
if (triggerResponse.StatusCode == System.Net.HttpStatusCode.OK)
{
job.UpdateBuildState(BuildState.PENDING);
ciService.Enqueue(job);
}
}
}
} catch (Exception e)
{
response.StatusCode = 500;