From baf45f4e004275508bb28baea4c6d406231b922c Mon Sep 17 00:00:00 2001 From: Harald Wolff Date: Thu, 3 Dec 2020 00:05:51 +0100 Subject: [PATCH] Beta 0.2.0 --- .gitignore | 5 + build.ln | 14 +- ln.build.server/Program.cs | 18 +- ln.build.server/ln.build.server.csproj | 2 +- ln.build/CIJob.cs | 53 +++-- ln.build/CIService.cs | 18 ++ ln.build/commands/CommandEnvironment.cs | 42 +++- ln.build/commands/CommandRunner.cs | 4 +- ln.build/ln.build.csproj | 2 +- ln.build/pipeline/DefaultPipeLine.cs | 184 ++++++++++++++++++ ln.build/pipeline/DotNetCommand.cs | 26 +++ ln.build/{ => pipeline}/DotNetPipeLine.cs | 1 - ln.build/{ => pipeline}/PipeLine.cs | 4 +- .../repositories/GitRepositoryInterface.cs | 12 +- .../repositories/GiteaRepositoryInterface.cs | 29 ++- 15 files changed, 356 insertions(+), 58 deletions(-) create mode 100644 ln.build/pipeline/DefaultPipeLine.cs create mode 100644 ln.build/pipeline/DotNetCommand.cs rename ln.build/{ => pipeline}/DotNetPipeLine.cs (99%) rename ln.build/{ => pipeline}/PipeLine.cs (95%) diff --git a/.gitignore b/.gitignore index bf793ed..542b474 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,8 @@ Thumbs.db # dotCover *.dotCover + +*.log +*.log.old +.vscode +.build \ No newline at end of file diff --git a/build.ln b/build.ln index 423f3f3..cc97e5f 100644 --- a/build.ln +++ b/build.ln @@ -25,21 +25,25 @@ { "name": "build", "commands": [ - "SH dotnet build -c Release" + "SH dotnet build -c $CONFIGURATION" ] }, { "name": "pack_and_publish", "commands": [ - "SH dotnet pack ln.build -o .build -c Release", - "SH dotnet publish ln.build.server -p:PublishTrimmed=true -p:PublishSingleFile=true -p:PublishReadyToRun=true --self-contained -r linux-x64 -c Release -o .build/linux-x64" + "SH dotnet pack ln.build -o .build -c $CONFIGURATION", + "SH dotnet pack ln.build.server -o .build -c $CONFIGURATION", + "SH dotnet publish ln.build.server -p:PublishTrimmed=true -p:PublishSingleFile=true -p:PublishReadyToRun=true --self-contained -r linux-x64 -c $CONFIGURATION -o .build/linux-x64" ] }, { "name": "push", "commands": [ - "SH dotnet nuget push .build/ln-build.*.nupkg -s ${NUGET_SOURCE} -k ${NUGET_APIKEY}" - ] + "SH dotnet nuget push .build/ln.build.*.nupkg -s $NUGET_SOURCE -k $NUGET_APIKEY" + ], + "secrets": { + "NUGET_APIKEY": "key/nuget.l--n.de" + } } ] } \ No newline at end of file diff --git a/ln.build.server/Program.cs b/ln.build.server/Program.cs index 6f18048..104aba4 100644 --- a/ln.build.server/Program.cs +++ b/ln.build.server/Program.cs @@ -18,6 +18,9 @@ namespace ln.build.server { static CIService CIService; + [StaticArgument( LongOption = "build")] + public static string BuildPath { get; set; } + static void Main(string[] args) { ArgumentContainer ac = new ArgumentContainer(typeof(Program)); @@ -27,10 +30,19 @@ namespace ln.build.server ac.Parse(ref args); CIService.Initialize(); - CIService.AddPipeline(new DotNetPipeLine()); - CIService.AddRepositoryInterface(new GiteaRepositoryInterface("https://git.l--n.de"){ AuthorizationToken = "1d03e9577c404b5b4f46b340147b1d500ff95b2e", }); - CIService.Start(); + if (BuildPath != null) + { + CIJob job = new CIJob(CIService,null); + job.WorkingDirectory = BuildPath; + job.RunJob(); + } else { + CIService.Initialize(); + CIService.AddRepositoryInterface(new GiteaRepositoryInterface("https://git.l--n.de"){ AuthorizationToken = "1d03e9577c404b5b4f46b340147b1d500ff95b2e", }); + + CIService.Start(); + } + } diff --git a/ln.build.server/ln.build.server.csproj b/ln.build.server/ln.build.server.csproj index 69f41ae..9fc36c9 100644 --- a/ln.build.server/ln.build.server.csproj +++ b/ln.build.server/ln.build.server.csproj @@ -6,7 +6,7 @@ - 0.1.0-test1 + 0.2.0 Harald Wolff-Thobaben l--n.de A simple build server scheduling builds triggered via web-hooks diff --git a/ln.build/CIJob.cs b/ln.build/CIJob.cs index 8d95625..60440df 100644 --- a/ln.build/CIJob.cs +++ b/ln.build/CIJob.cs @@ -4,6 +4,7 @@ using System.IO; using System.Net.Http; using System.Text; using ln.build.commands; +using ln.build.pipeline; using ln.build.repositories; using ln.json; using ln.logging; @@ -24,7 +25,7 @@ namespace ln.build public RepositoryInterface RepositoryInterface { get; set; } - List pipeLines = new List(); + public DefaultPipeLine PipeLine { get; set; } public BuildState BuildState { get; private set; } @@ -39,7 +40,9 @@ namespace ln.build RepositoryInterface = repositoryInterface; WorkingDirectory = Path.Combine(Path.GetTempPath(), JobID); - Logger = new Logger(new FileLogger(Path.Combine(Path.GetTempPath(), String.Format("{0}.log", JobID)))); + Directory.CreateDirectory(Path.Combine(ciService.ReportsDirectory, JobID)); + + Logger = new Logger(new FileLogger(Path.Combine(ciService.ReportsDirectory, JobID, "build.log"))); Logger.Backends.Add(Logger.ConsoleLogger); RepositoryURL = repositoryURL; @@ -90,20 +93,17 @@ namespace ln.build { if (CloneRepository()) { - if (DetectPipelines()) + try { - UpdateBuildState(BuildState.PENDING); + LoadPipeLine(); - try{ - foreach (PipeLine pipeLine in pipeLines) - { - pipeLine.Run(this); - } - UpdateBuildState(BuildState.SUCCESS); - } catch (Exception e) - { - UpdateBuildState(BuildState.FAILURE); - } + PipeLine.Run(); + + UpdateBuildState(BuildState.SUCCESS); + } catch (Exception e) + { + Logger.Log(e); + UpdateBuildState(BuildState.FAILURE); } } else { Logger.Log(LogLevel.ERROR, "CIJob failed at CloneRepository()"); @@ -115,23 +115,20 @@ namespace ln.build public bool CloneRepository() { - Logging.Log("cloning repository to {0}", WorkingDirectory); - return - (new CommandRunner(Logger, "git", "clone", RepositoryURL, WorkingDirectory).Run(Environment) == 0) && - (!ContainsVariable("COMMIT_ID") || new CommandRunner(Logger, "git", "checkout", GetVariable("COMMIT_ID")){ WorkingDirectory = this.WorkingDirectory, }.Run(Environment) == 0); + RepositoryInterface?.CloneSources(this); + return true; } - public bool DetectPipelines() + public void LoadPipeLine() { - if (CIService != null) + JSONObject jsonPipeLine = JSONParser.ParseFile(Path.Combine(WorkingDirectory, "build.ln")) as JSONObject; + if (jsonPipeLine != null) { - foreach (PipeLine pipeLine in CIService.PipeLines) - { - if (pipeLine.DetectValidity(this)) - pipeLines.Add(pipeLine); - } + DefaultPipeLine pipeLine = new DefaultPipeLine(CIService, Environment); + pipeLine.LoadJson(jsonPipeLine); + + PipeLine = pipeLine; } - return pipeLines.Count > 0; } public void Notify() @@ -150,7 +147,9 @@ namespace ln.build public void Cleanup() { CIService.CreateReport(this); - Directory.Delete(WorkingDirectory, true); + + if (RepositoryInterface != null) + Directory.Delete(WorkingDirectory, true); } } diff --git a/ln.build/CIService.cs b/ln.build/CIService.cs index ed436a6..1e9ecc5 100644 --- a/ln.build/CIService.cs +++ b/ln.build/CIService.cs @@ -44,9 +44,26 @@ namespace ln.build HashSet pipelines = new HashSet(); public IEnumerable PipeLines => pipelines; + Dictionary secrets = new Dictionary(); + + public CIService() { buildPool = new Pool(2); + + String secretsPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),".ln.build.secrets.json"); + Logging.Log(LogLevel.INFO, "Trying to load secrets from {0}", secretsPath); + if (File.Exists(secretsPath)) + { + JSONObject secretsObject = JSONParser.ParseFile(secretsPath) as JSONObject; + foreach (string key in secretsObject.Keys) + { + secrets.Add(key, secretsObject[key].ToNative().ToString()); + } + Logging.Log(LogLevel.INFO, "Secrets loaded from {0}", secretsPath); + } + + } public CIService(string baseDirectory) : this() { @@ -113,6 +130,7 @@ namespace ln.build public string GetJobURL(CIJob job) => string.Format("{0}/builds/{1}", BaseURL, job.JobID); + public string GetSecret(string key) => secrets[key]; public void Start() { diff --git a/ln.build/commands/CommandEnvironment.cs b/ln.build/commands/CommandEnvironment.cs index 0c534c0..8f644c0 100644 --- a/ln.build/commands/CommandEnvironment.cs +++ b/ln.build/commands/CommandEnvironment.cs @@ -3,12 +3,19 @@ using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using ln.json; using ln.logging; namespace ln.build.commands { public class CommandEnvironment : IDictionary { + public CommandEnvironment Parent { get; } + + public string WorkingDirectory { get; set; } = Path.GetFullPath("."); + static string[] DefaultVariables = { "PATH", "HOME", "USERNAME" }; public Logger Logger { get; set; } Dictionary variables = new Dictionary(); @@ -19,14 +26,16 @@ namespace ln.build.commands public bool IsReadOnly => ((ICollection>)variables).IsReadOnly; public string this[string key] { get => Get(key); set => Set(key,value); } + public CommandEnvironment(CommandEnvironment parent) : this(parent.Logger) { Parent = parent; WorkingDirectory = parent.WorkingDirectory; } + public CommandEnvironment(CommandEnvironment parent, Logger logger) : this(logger) { Parent = parent; WorkingDirectory = parent.WorkingDirectory; } public CommandEnvironment() : this(Logger.Default) - { - foreach (string defName in DefaultVariables) - Set(defName, Environment.GetEnvironmentVariable(defName)); - } + { } public CommandEnvironment(Logger logger) { Logger = logger; + + foreach (string defName in DefaultVariables) + Set(defName, Environment.GetEnvironmentVariable(defName)); } public CommandEnvironment(IEnumerable> setup) : this(Logger.Default, setup) { } public CommandEnvironment(Logger logger, IEnumerable> setup) : this(logger) @@ -40,7 +49,12 @@ namespace ln.build.commands public string Get(string varName,string defValue) { if (!variables.TryGetValue(varName,out string value)) - value = defValue; + { + if (Parent != null) + value = Parent.Get(varName, defValue); + else + value = defValue; + } return value; } public CommandEnvironment Set(string varName,string value) @@ -123,9 +137,25 @@ namespace ln.build.commands public IEnumerator> GetEnumerator() { - return ((IEnumerable>)variables).GetEnumerator(); + if (Parent != null) + { + return Keys.Concat(Parent.Keys).Distinct().Select((k)=>new KeyValuePair(k,Get(k))).GetEnumerator(); + } else + { + return ((IEnumerable>)variables).GetEnumerator(); + } } + + public void Apply(JSONObject jsonEnvironment) + { + foreach (string key in jsonEnvironment?.Keys ?? new string[0]) + { + Set(key, jsonEnvironment[key].ToNative().ToString()); + } + } + + IEnumerator IEnumerable.GetEnumerator() { return ((IEnumerable)variables).GetEnumerator(); diff --git a/ln.build/commands/CommandRunner.cs b/ln.build/commands/CommandRunner.cs index 34cc7eb..255eb08 100644 --- a/ln.build/commands/CommandRunner.cs +++ b/ln.build/commands/CommandRunner.cs @@ -18,8 +18,6 @@ namespace ln.build.commands List arguments = new List(); public IEnumerable Arguments => arguments; - public string WorkingDirectory { get; set; } = "."; - public Logger Logger { get; } Func TestExitCode = null; @@ -67,7 +65,7 @@ namespace ln.build.commands RedirectStandardError = true, RedirectStandardOutput = true, RedirectStandardInput = true, - WorkingDirectory = this.WorkingDirectory, + WorkingDirectory = environment.WorkingDirectory }; psi.Environment.Clear(); diff --git a/ln.build/ln.build.csproj b/ln.build/ln.build.csproj index bb2796d..f5013da 100644 --- a/ln.build/ln.build.csproj +++ b/ln.build/ln.build.csproj @@ -5,7 +5,7 @@ - 0.1.0-test2 + 0.2.0 Harald Wolff-Thobaben l--n.de A simple build server scheduling builds triggered via web-hooks diff --git a/ln.build/pipeline/DefaultPipeLine.cs b/ln.build/pipeline/DefaultPipeLine.cs new file mode 100644 index 0000000..0df063f --- /dev/null +++ b/ln.build/pipeline/DefaultPipeLine.cs @@ -0,0 +1,184 @@ + + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using ln.build.commands; +using ln.json; +using ln.logging; + +namespace ln.build.pipeline +{ + + public class DefaultPipeLine + { + public CommandEnvironment CommandEnvironment { get; } + public CIService CIService { get; } + + List stages = new List(); + public IEnumerable Stages => stages; + + public DefaultPipeLine(CIService ciService) + { + CIService = ciService; + CommandEnvironment = new CommandEnvironment(); + } + public DefaultPipeLine(CIService ciService, CommandEnvironment commandEnvironment) + { + CIService = ciService; + CommandEnvironment = new CommandEnvironment(commandEnvironment); + } + + public void LoadJson(JSONObject jsonPipeLine) + { + if (jsonPipeLine.ContainsKey("env")) + CommandEnvironment.Apply(jsonPipeLine["env"] as JSONObject); + + if (jsonPipeLine.ContainsKey("stages")) + { + JSONArray jsonStages = jsonPipeLine["stages"] as JSONArray; + foreach (JSONObject jsonStage in jsonStages.Children) + { + Stage stage = new Stage(this); + stage.LoadJson(jsonStage); + + stages.Add(stage); + } + } + } + + public void Run() + { + foreach (Stage stage in stages) + { + CommandEnvironment.Logger.Log(LogLevel.INFO,"-------------------------------------------------------------------------------------"); + CommandEnvironment.Logger.Log(LogLevel.INFO,"STAGE: {0}", stage.Name); + CommandEnvironment.Logger.Log(LogLevel.INFO,"-------------------------------------------------------------------------------------"); + stage.Run(); + } + } + + + } + + public class Stage + { + public DefaultPipeLine PipeLine { get; } + public string Name { get; set; } + + public CommandEnvironment CommandEnvironment { get; } + + List commands = new List(); + public IEnumerable Commands => commands; + + public Stage(DefaultPipeLine pipeLine) + { + PipeLine = pipeLine; + CommandEnvironment = new CommandEnvironment(pipeLine.CommandEnvironment); + } + + public void LoadJson(JSONObject jsonStage) + { + Name = jsonStage["name"].ToNative().ToString(); + + if (jsonStage.ContainsKey("env")) + { + CommandEnvironment.Apply(jsonStage["env"] as JSONObject); + } + if (jsonStage.ContainsKey("commands")) + { + JSONArray jsonCommands = jsonStage["commands"] as JSONArray; + foreach (JSONValue jsonValue in jsonCommands.Children) + { + commands.Add(StageCommand.Create(jsonValue.ToNative().ToString())); + } + } + if (jsonStage.ContainsKey("secrets")) + { + JSONObject jsonSecrets = jsonStage["secrets"] as JSONObject; + foreach (string key in jsonSecrets.Keys) + { + CommandEnvironment.Set(key, PipeLine.CIService.GetSecret(jsonSecrets[key].ToNative().ToString())); + } + + } + } + + + public void Run() + { + foreach (StageCommand command in commands) + command.Run(this); + } + } + + public abstract class StageCommand + { + public string Name { get; } + + public StageCommand(string name) + { + Name = name; + } + + public abstract void Run(Stage stage); + + + + + + static Dictionary> commandFactories = new Dictionary>(); + + public static StageCommand Create(string cmdline) + { + string[] tokens = cmdline.Split(new char[]{' ','\t'}, 2); + + if (commandFactories.TryGetValue(tokens[0],out Func factory)) + { + return factory(tokens[1]); + } + throw new Exception(string.Format("can't find factory for command keyword '{0}'", tokens[0])); + } + + static StageCommand() + { + commandFactories.Add("SH", (args) => new ShellCommand(args)); + commandFactories.Add("DOTNET", (args) => new DotNetCommand(args)); + commandFactories.Add("GITEA", (args) => new DotNetCommand(args)); + + } + + } + + public class ShellCommand : StageCommand + { + public event CommandExitedDelegate OnCommandExited; + + public CommandRunner CommandRunner { get; } + + public ShellCommand(string filename,params CommandRunner.Argument[] arguments) + :base("SH") + { + CommandRunner = new CommandRunner("/bin/bash", "-c"){ Throw = CRThrow.NEVER, }; + CommandRunner.AddArguments(arguments); + } + public ShellCommand(string cmdline) + :base("SH") + { + CommandRunner = new CommandRunner("/bin/bash", "-c", string.Format("\"{0}\"",cmdline)){ Throw = CRThrow.NEVER, }; + } + + public override void Run(Stage stage) + { + MemoryStream logStream = new MemoryStream(); + int result = CommandRunner.Run(stage.CommandEnvironment, logStream); + stage.CommandEnvironment.Logger.Log(LogLevel.INFO, "command output:\n{0}", Encoding.UTF8.GetString(logStream.ToArray())); + stage.CommandEnvironment.Logger.Log(LogLevel.INFO, "command exit code: {0}", result); + if (result != 0) + throw new Exception(String.Format("command exited with code {0} [0x{0:x}]", result)); + } + } + +} \ No newline at end of file diff --git a/ln.build/pipeline/DotNetCommand.cs b/ln.build/pipeline/DotNetCommand.cs new file mode 100644 index 0000000..43e7499 --- /dev/null +++ b/ln.build/pipeline/DotNetCommand.cs @@ -0,0 +1,26 @@ + +using ln.build.pipeline; + +namespace ln.build.commands +{ + + public class DotNetCommand : StageCommand + { + + + public DotNetCommand(string arguments) :base("DOTNET"){ + + } + + public void Analyze(Stage stage) + { + } + + + public override void Run(Stage stage) + { + + } + } + +} \ No newline at end of file diff --git a/ln.build/DotNetPipeLine.cs b/ln.build/pipeline/DotNetPipeLine.cs similarity index 99% rename from ln.build/DotNetPipeLine.cs rename to ln.build/pipeline/DotNetPipeLine.cs index c864b7d..e020211 100644 --- a/ln.build/DotNetPipeLine.cs +++ b/ln.build/pipeline/DotNetPipeLine.cs @@ -114,7 +114,6 @@ namespace ln.build return success ? 0 : -1; } } - } diff --git a/ln.build/PipeLine.cs b/ln.build/pipeline/PipeLine.cs similarity index 95% rename from ln.build/PipeLine.cs rename to ln.build/pipeline/PipeLine.cs index 691b3fd..464a880 100644 --- a/ln.build/PipeLine.cs +++ b/ln.build/pipeline/PipeLine.cs @@ -63,9 +63,7 @@ namespace ln.build } public override int Run(CIJob job) - { - CommandRunner.WorkingDirectory = job.WorkingDirectory; - + { int result = CommandRunner.Run(job.Environment, job.GetLogStream(DefaultLogFileName)); OnCommandExited?.Invoke(this, job, result); return result; diff --git a/ln.build/repositories/GitRepositoryInterface.cs b/ln.build/repositories/GitRepositoryInterface.cs index 045f063..77f7b11 100644 --- a/ln.build/repositories/GitRepositoryInterface.cs +++ b/ln.build/repositories/GitRepositoryInterface.cs @@ -1,5 +1,6 @@ using System; +using System.IO; using ln.build.commands; using ln.http; @@ -18,8 +19,15 @@ namespace ln.build.repositories public override void CloneSources(CIJob job) { job.Logger.Log("{0}: cloning repository to {1}", GetType().Name, job.WorkingDirectory); - bool success = (new CommandRunner("git", "clone", job.RepositoryURL, job.WorkingDirectory).Run(job.Environment) == 0) && - (!job.ContainsVariable("COMMIT_ID") || new CommandRunner("git", "checkout", job.GetVariable("COMMIT_ID")){ WorkingDirectory = job.WorkingDirectory, }.Run(job.Environment) == 0); + job.Environment.WorkingDirectory = Path.GetTempPath(); + + bool success = new CommandRunner("git", "clone", job.RepositoryURL, job.WorkingDirectory).Run(job.Environment) == 0; + job.Environment.WorkingDirectory = job.WorkingDirectory; + + if (success && job.ContainsVariable("COMMIT_ID")) + { + success = new CommandRunner("git", "checkout", job.GetVariable("COMMIT_ID")).Run(job.Environment) == 0; + } if (!success) throw new Exception("clone failed"); } diff --git a/ln.build/repositories/GiteaRepositoryInterface.cs b/ln.build/repositories/GiteaRepositoryInterface.cs index f8fbd33..a169a66 100644 --- a/ln.build/repositories/GiteaRepositoryInterface.cs +++ b/ln.build/repositories/GiteaRepositoryInterface.cs @@ -19,6 +19,13 @@ namespace ln.build.repositories BaseURL = baseURL; } + HttpClient CreateHttpClient() + { + HttpClient httpClient = new HttpClient(); + httpClient.DefaultRequestHeaders.Add("Authorization",String.Format("token {0}", AuthorizationToken)); + return httpClient; + } + public override bool DetectValidity(string cloneUrl) => cloneUrl.StartsWith(BaseURL); public override void UpdateBuildState(CIJob job) { @@ -37,10 +44,8 @@ namespace ln.build.repositories stateObject.Add("state", job.BuildState.ToString().ToLower()); stateObject.Add("target_url", job.CIService.GetJobURL(job)); - using (HttpClient httpClient = new HttpClient()) + using (HttpClient httpClient = CreateHttpClient()) { - httpClient.DefaultRequestHeaders.Add("Authorization",String.Format("token {0}", AuthorizationToken)); - HttpResponseMessage response = httpClient.PostAsync(buildStateURL, new StringContent(stateObject.ToString(),Encoding.UTF8,"application/json")).Result; job.Logger.Log(LogLevel.DEBUG, "UpdateBuildState({0}): {1}", job.BuildState, buildStateURL); job.Logger.Log(LogLevel.DEBUG, "Request: {0}", stateObject.ToString()); @@ -82,11 +87,23 @@ namespace ln.build.repositories .SetVariable("NUGET_SOURCE", "http://nuget.l--n.de/nuget/l--n/v3/index.json") ; - job.UpdateBuildState(BuildState.PENDING); + using (HttpClient httpClient = CreateHttpClient()) + { + string triggerFile = string.Format("{3}/api/v1/repos/{0}/{1}/contents/build.ln?ref={2}", + job.GetVariable("REPO_OWNER"), + job.GetVariable("REPO_NAME"), + job.GetVariable("COMMIT_ID"), + BaseURL + ); - ciService.Enqueue(job); + HttpResponseMessage triggerResponse = httpClient.GetAsync(triggerFile).Result; + if (triggerResponse.StatusCode == System.Net.HttpStatusCode.OK) + { + job.UpdateBuildState(BuildState.PENDING); + ciService.Enqueue(job); + } + } } - } catch (Exception e) { response.StatusCode = 500;