Beta 0.2.0

.gitignore

@@ -39,3 +39,8 @@ Thumbs.db
 
 # dotCover
 *.dotCover
+
+*.log
+*.log.old
+.vscode
+.build

build.ln

@@ -25,21 +25,25 @@
         {
             "name": "build",
             "commands": [
-                "SH dotnet build -c Release"
+                "SH dotnet build -c $CONFIGURATION"
             ]
         },
         {
             "name": "pack_and_publish",
             "commands": [
-                "SH dotnet pack ln.build -o .build -c Release",
+                "SH dotnet pack ln.build -o .build -c $CONFIGURATION",
-                "SH dotnet publish ln.build.server -p:PublishTrimmed=true -p:PublishSingleFile=true -p:PublishReadyToRun=true --self-contained -r linux-x64 -c Release -o .build/linux-x64"
+                "SH dotnet pack ln.build.server -o .build -c $CONFIGURATION",
+                "SH dotnet publish ln.build.server -p:PublishTrimmed=true -p:PublishSingleFile=true -p:PublishReadyToRun=true --self-contained -r linux-x64 -c $CONFIGURATION -o .build/linux-x64"
             ]
         },
         {
             "name": "push",
             "commands": [
-                "SH dotnet nuget push .build/ln-build.*.nupkg -s ${NUGET_SOURCE} -k ${NUGET_APIKEY}"
+                "SH dotnet nuget push .build/ln.build.*.nupkg -s $NUGET_SOURCE -k $NUGET_APIKEY"
-            ]
+            ],
+            "secrets": {
+                "NUGET_APIKEY": "key/nuget.l--n.de"
+            }
         }
     ]
 }

ln.build.server/Program.cs

@@ -18,6 +18,9 @@ namespace ln.build.server
     {
         static CIService CIService;
 
+        [StaticArgument( LongOption = "build")]
+        public static string BuildPath { get; set; }
+
         static void Main(string[] args)
         {
             ArgumentContainer ac = new ArgumentContainer(typeof(Program));
@@ -27,10 +30,19 @@ namespace ln.build.server
             ac.Parse(ref args);
 
             CIService.Initialize();
-            CIService.AddPipeline(new DotNetPipeLine());
-            CIService.AddRepositoryInterface(new GiteaRepositoryInterface("https://git.l--n.de"){ AuthorizationToken = "1d03e9577c404b5b4f46b340147b1d500ff95b2e", });
 
-            CIService.Start();
+            if (BuildPath != null)
+            {
+                CIJob job = new CIJob(CIService,null);
+                job.WorkingDirectory = BuildPath;
+                job.RunJob();
+            } else {
+                CIService.Initialize();
+                CIService.AddRepositoryInterface(new GiteaRepositoryInterface("https://git.l--n.de"){ AuthorizationToken = "1d03e9577c404b5b4f46b340147b1d500ff95b2e", });
+
+                CIService.Start();
+            }
+
         }
 
 

ln.build.server/ln.build.server.csproj

@@ -6,7 +6,7 @@
   </PropertyGroup>
 
   <PropertyGroup>
-    <Version>0.1.0-test1</Version>
+    <Version>0.2.0</Version>
     <Authors>Harald Wolff-Thobaben</Authors>
     <Company>l--n.de</Company>
     <Description>A simple build server scheduling builds triggered via web-hooks</Description>

ln.build/CIJob.cs

@@ -4,6 +4,7 @@ using System.IO;
 using System.Net.Http;
 using System.Text;
 using ln.build.commands;
+using ln.build.pipeline;
 using ln.build.repositories;
 using ln.json;
 using ln.logging;
@@ -24,7 +25,7 @@ namespace ln.build
 
         public RepositoryInterface RepositoryInterface { get; set; }
 
-        List<PipeLine> pipeLines = new List<PipeLine>();
+        public DefaultPipeLine PipeLine { get; set; }
         
         public BuildState BuildState { get; private set; }
 
@@ -39,7 +40,9 @@ namespace ln.build
             RepositoryInterface = repositoryInterface;
 
             WorkingDirectory = Path.Combine(Path.GetTempPath(), JobID);
-            Logger = new Logger(new FileLogger(Path.Combine(Path.GetTempPath(), String.Format("{0}.log", JobID))));
+            Directory.CreateDirectory(Path.Combine(ciService.ReportsDirectory, JobID));
+
+            Logger = new Logger(new FileLogger(Path.Combine(ciService.ReportsDirectory, JobID, "build.log")));
             Logger.Backends.Add(Logger.ConsoleLogger);
 
             RepositoryURL = repositoryURL;
@@ -90,20 +93,17 @@ namespace ln.build
         {
             if (CloneRepository())
             {
-                if (DetectPipelines())
+                try
                 {
-                    UpdateBuildState(BuildState.PENDING);
+                    LoadPipeLine();
 
-                    try{
+                    PipeLine.Run();
-                        foreach (PipeLine pipeLine in pipeLines)
+
-                        {
+                    UpdateBuildState(BuildState.SUCCESS);
-                            pipeLine.Run(this);
+                } catch (Exception e)
-                        }
+                {
-                        UpdateBuildState(BuildState.SUCCESS);
+                    Logger.Log(e);
-                    } catch (Exception e)
+                    UpdateBuildState(BuildState.FAILURE);
-                    {
-                        UpdateBuildState(BuildState.FAILURE);
-                    }
                 }
             } else {
                 Logger.Log(LogLevel.ERROR, "CIJob failed at CloneRepository()");
@@ -115,23 +115,20 @@ namespace ln.build
 
         public bool CloneRepository()
         {
-            Logging.Log("cloning repository to {0}", WorkingDirectory);
+            RepositoryInterface?.CloneSources(this);
-            return 
+            return true;
-                (new CommandRunner(Logger, "git", "clone", RepositoryURL, WorkingDirectory).Run(Environment) == 0) &&
-                (!ContainsVariable("COMMIT_ID") || new CommandRunner(Logger, "git", "checkout", GetVariable("COMMIT_ID")){ WorkingDirectory = this.WorkingDirectory, }.Run(Environment) == 0);
         }
 
-        public bool DetectPipelines()
+        public void LoadPipeLine()
         {
-            if (CIService != null)
+            JSONObject jsonPipeLine = JSONParser.ParseFile(Path.Combine(WorkingDirectory, "build.ln")) as JSONObject;
+            if (jsonPipeLine != null)
             {
-                foreach (PipeLine pipeLine in CIService.PipeLines)
+                DefaultPipeLine pipeLine = new DefaultPipeLine(CIService, Environment);
-                {
+                pipeLine.LoadJson(jsonPipeLine);
-                    if (pipeLine.DetectValidity(this))
+
-                        pipeLines.Add(pipeLine);
+                PipeLine = pipeLine;
-                }
             }
-            return pipeLines.Count > 0;
         }
 
         public void Notify()
@@ -150,7 +147,9 @@ namespace ln.build
         public void Cleanup()
         {
             CIService.CreateReport(this);
-            Directory.Delete(WorkingDirectory, true);
+
+            if (RepositoryInterface != null)
+                Directory.Delete(WorkingDirectory, true);
         }
 
     }

ln.build/CIService.cs

@@ -44,9 +44,26 @@ namespace ln.build
         HashSet<PipeLine> pipelines = new HashSet<PipeLine>();
         public IEnumerable<PipeLine> PipeLines => pipelines;
 
+        Dictionary<string,string> secrets = new Dictionary<string, string>();
+
+
         public CIService()
         {
             buildPool = new Pool(2);
+
+            String secretsPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),".ln.build.secrets.json");
+            Logging.Log(LogLevel.INFO, "Trying to load secrets from {0}", secretsPath);
+            if (File.Exists(secretsPath))
+            {
+                JSONObject secretsObject = JSONParser.ParseFile(secretsPath) as JSONObject;
+                foreach (string key in secretsObject.Keys)
+                {
+                    secrets.Add(key, secretsObject[key].ToNative().ToString());
+                }
+                Logging.Log(LogLevel.INFO, "Secrets loaded from {0}", secretsPath);
+            }
+
+
         }
         public CIService(string baseDirectory) : this()
         {
@@ -113,6 +130,7 @@ namespace ln.build
 
         public string GetJobURL(CIJob job) => string.Format("{0}/builds/{1}", BaseURL, job.JobID);
 
+        public string GetSecret(string key) => secrets[key];
 
         public void Start()
         {

ln.build/commands/CommandEnvironment.cs

@@ -3,12 +3,19 @@ using System;
 using System.Collections;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
+using System.IO;
+using System.Linq;
+using ln.json;
 using ln.logging;
 
 namespace ln.build.commands
 {
     public class CommandEnvironment : IDictionary<string,string>
     {
+        public CommandEnvironment Parent { get; }
+
+        public string WorkingDirectory { get; set; } = Path.GetFullPath(".");
+
         static string[] DefaultVariables = { "PATH", "HOME", "USERNAME" };
         public Logger Logger { get; set; }
         Dictionary<string,string> variables = new Dictionary<string, string>();
@@ -19,14 +26,16 @@ namespace ln.build.commands
         public bool IsReadOnly => ((ICollection<KeyValuePair<string, string>>)variables).IsReadOnly;
         public string this[string key] { get => Get(key); set => Set(key,value); }
 
+        public CommandEnvironment(CommandEnvironment parent) : this(parent.Logger) { Parent = parent; WorkingDirectory = parent.WorkingDirectory; }
+        public CommandEnvironment(CommandEnvironment parent, Logger logger) : this(logger) { Parent = parent; WorkingDirectory = parent.WorkingDirectory; }
         public CommandEnvironment() : this(Logger.Default)
-        {
+        { }
-            foreach (string defName in DefaultVariables)
-                Set(defName, Environment.GetEnvironmentVariable(defName));
-        }
         public CommandEnvironment(Logger logger)
         { 
             Logger = logger;
+
+            foreach (string defName in DefaultVariables)
+                Set(defName, Environment.GetEnvironmentVariable(defName));
         }
         public CommandEnvironment(IEnumerable<KeyValuePair<string,string>> setup) : this(Logger.Default, setup) { }
         public CommandEnvironment(Logger logger, IEnumerable<KeyValuePair<string,string>> setup) : this(logger)
@@ -40,7 +49,12 @@ namespace ln.build.commands
         public string Get(string varName,string defValue)
         {
             if (!variables.TryGetValue(varName,out string value))
-                value = defValue;
+            {
+                if (Parent != null)
+                    value = Parent.Get(varName, defValue);
+                else
+                    value = defValue;
+            }
             return value;
         }
         public CommandEnvironment Set(string varName,string value)
@@ -123,9 +137,25 @@ namespace ln.build.commands
 
         public IEnumerator<KeyValuePair<string, string>> GetEnumerator()
         {
-            return ((IEnumerable<KeyValuePair<string, string>>)variables).GetEnumerator();
+            if (Parent != null)
+            {
+                return Keys.Concat(Parent.Keys).Distinct().Select((k)=>new KeyValuePair<string, string>(k,Get(k))).GetEnumerator();
+            } else 
+            {
+                return ((IEnumerable<KeyValuePair<string, string>>)variables).GetEnumerator();
+            }
         }
 
+
+        public void Apply(JSONObject jsonEnvironment)
+        {
+            foreach (string key in jsonEnvironment?.Keys ?? new string[0])
+            {
+                Set(key, jsonEnvironment[key].ToNative().ToString());
+            }
+        }
+
+
         IEnumerator IEnumerable.GetEnumerator()
         {
             return ((IEnumerable)variables).GetEnumerator();

ln.build/commands/CommandRunner.cs

@@ -18,8 +18,6 @@ namespace ln.build.commands
         List<Argument> arguments = new List<Argument>();
         public IEnumerable<Argument> Arguments => arguments;
 
-        public string WorkingDirectory { get; set; } = ".";
-
         public Logger Logger { get; }
 
         Func<int,bool> TestExitCode = null;
@@ -67,7 +65,7 @@ namespace ln.build.commands
                 RedirectStandardError = true,
                 RedirectStandardOutput = true,
                 RedirectStandardInput = true,
-                WorkingDirectory = this.WorkingDirectory,
+                WorkingDirectory = environment.WorkingDirectory
             };
 
             psi.Environment.Clear();

ln.build/ln.build.csproj

@@ -5,7 +5,7 @@
   </PropertyGroup>
 
   <PropertyGroup>
-    <Version>0.1.0-test2</Version>
+    <Version>0.2.0</Version>
     <Authors>Harald Wolff-Thobaben</Authors>
     <Company>l--n.de</Company>
     <Description>A simple build server scheduling builds triggered via web-hooks</Description>

ln.build/pipeline/DefaultPipeLine.cs

@@ -0,0 +1,184 @@
+
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using ln.build.commands;
+using ln.json;
+using ln.logging;
+
+namespace ln.build.pipeline
+{
+
+    public class DefaultPipeLine
+    {
+        public CommandEnvironment CommandEnvironment { get; }
+        public CIService CIService { get; }
+
+        List<Stage> stages = new List<Stage>();
+        public IEnumerable<Stage> Stages => stages;
+
+        public DefaultPipeLine(CIService ciService)
+        {
+            CIService = ciService;
+            CommandEnvironment = new CommandEnvironment();
+        }
+        public DefaultPipeLine(CIService ciService, CommandEnvironment commandEnvironment)
+        {
+            CIService = ciService;
+            CommandEnvironment = new CommandEnvironment(commandEnvironment);
+        }
+
+        public void LoadJson(JSONObject jsonPipeLine)
+        {
+            if (jsonPipeLine.ContainsKey("env"))
+                CommandEnvironment.Apply(jsonPipeLine["env"] as JSONObject);
+
+            if (jsonPipeLine.ContainsKey("stages"))
+            {
+                JSONArray jsonStages = jsonPipeLine["stages"] as JSONArray;
+                foreach (JSONObject jsonStage in jsonStages.Children)
+                {
+                    Stage stage = new Stage(this);
+                    stage.LoadJson(jsonStage);
+
+                    stages.Add(stage);
+                }
+            }
+        }
+
+        public void Run()
+        {
+            foreach (Stage stage in stages)
+            {
+                CommandEnvironment.Logger.Log(LogLevel.INFO,"-------------------------------------------------------------------------------------");
+                CommandEnvironment.Logger.Log(LogLevel.INFO,"STAGE: {0}", stage.Name);
+                CommandEnvironment.Logger.Log(LogLevel.INFO,"-------------------------------------------------------------------------------------");
+                stage.Run();
+            }
+        }
+
+
+    }
+
+    public class Stage
+    {
+        public DefaultPipeLine PipeLine { get; }
+        public string Name { get; set; }
+
+        public CommandEnvironment CommandEnvironment { get; }
+
+        List<StageCommand> commands = new List<StageCommand>();
+        public IEnumerable<StageCommand> Commands => commands;
+
+        public Stage(DefaultPipeLine pipeLine)
+        {
+            PipeLine = pipeLine;
+            CommandEnvironment = new CommandEnvironment(pipeLine.CommandEnvironment);
+        }
+
+        public void LoadJson(JSONObject jsonStage)
+        {
+            Name = jsonStage["name"].ToNative().ToString();
+
+            if (jsonStage.ContainsKey("env"))
+            {
+                CommandEnvironment.Apply(jsonStage["env"] as JSONObject);
+            }
+            if (jsonStage.ContainsKey("commands"))
+            {
+                JSONArray jsonCommands = jsonStage["commands"] as JSONArray;
+                foreach (JSONValue jsonValue in jsonCommands.Children)
+                {
+                    commands.Add(StageCommand.Create(jsonValue.ToNative().ToString()));
+                }
+            }
+            if (jsonStage.ContainsKey("secrets"))
+            {
+                JSONObject jsonSecrets = jsonStage["secrets"] as JSONObject;
+                foreach (string key in jsonSecrets.Keys)
+                {
+                    CommandEnvironment.Set(key, PipeLine.CIService.GetSecret(jsonSecrets[key].ToNative().ToString()));
+                }
+
+            }
+        }
+
+
+        public void Run()
+        {
+            foreach (StageCommand command in commands)
+                command.Run(this);
+        }
+    }
+
+    public abstract class StageCommand
+    {
+        public string Name { get; }
+
+        public StageCommand(string name)
+        {
+            Name = name;
+        }
+
+        public abstract void Run(Stage stage);
+
+
+
+
+
+        static Dictionary<string,Func<string,StageCommand>> commandFactories = new Dictionary<string, Func<string, StageCommand>>();
+
+        public static StageCommand Create(string cmdline)
+        {
+            string[] tokens = cmdline.Split(new char[]{' ','\t'}, 2);
+            
+            if (commandFactories.TryGetValue(tokens[0],out Func<string,StageCommand> factory))
+            {
+                return factory(tokens[1]);
+            }
+            throw new Exception(string.Format("can't find factory for command keyword '{0}'", tokens[0]));
+        }
+
+        static StageCommand()
+        {
+            commandFactories.Add("SH", (args) => new ShellCommand(args));
+            commandFactories.Add("DOTNET", (args) => new DotNetCommand(args));
+            commandFactories.Add("GITEA", (args) => new DotNetCommand(args));
+
+        }
+
+    }
+
+    public class ShellCommand : StageCommand
+    {
+        public event CommandExitedDelegate OnCommandExited;
+
+        public CommandRunner CommandRunner { get; }
+
+        public ShellCommand(string filename,params CommandRunner.Argument[] arguments)
+        :base("SH")
+        {
+            CommandRunner = new CommandRunner("/bin/bash", "-c"){ Throw = CRThrow.NEVER, };
+            CommandRunner.AddArguments(arguments);
+        }
+        public ShellCommand(string cmdline)
+        :base("SH")
+        {
+            CommandRunner = new CommandRunner("/bin/bash", "-c", string.Format("\"{0}\"",cmdline)){ Throw = CRThrow.NEVER, };
+        }
+
+        public override void Run(Stage stage)
+        {
+            MemoryStream logStream = new MemoryStream();
+            int result = CommandRunner.Run(stage.CommandEnvironment, logStream);
+            stage.CommandEnvironment.Logger.Log(LogLevel.INFO, "command output:\n{0}", Encoding.UTF8.GetString(logStream.ToArray()));
+            stage.CommandEnvironment.Logger.Log(LogLevel.INFO, "command exit code: {0}", result);
+            if (result != 0)
+                throw new Exception(String.Format("command exited with code {0} [0x{0:x}]", result));
+        }
+    }
+
+}

ln.build/pipeline/DotNetCommand.cs

@@ -0,0 +1,26 @@
+
+using ln.build.pipeline;
+
+namespace ln.build.commands
+{
+
+    public class DotNetCommand : StageCommand
+    {
+        
+
+        public DotNetCommand(string arguments) :base("DOTNET"){
+
+        }
+
+        public void Analyze(Stage stage)
+        {
+        }
+
+
+        public override void Run(Stage stage)
+        {
+            
+        }
+    }
+
+}

ln.build/pipeline/DotNetPipeLine.cs

@@ -114,7 +114,6 @@ namespace ln.build
                 return success ? 0 : -1;
             }
         }
-        
 
     }
 

ln.build/pipeline/PipeLine.cs

@@ -63,9 +63,7 @@ namespace ln.build
         }
 
         public override int Run(CIJob job)
-        {
+        {            
-            CommandRunner.WorkingDirectory = job.WorkingDirectory;
-            
             int result = CommandRunner.Run(job.Environment, job.GetLogStream(DefaultLogFileName));
             OnCommandExited?.Invoke(this, job, result);
             return result;

ln.build/repositories/GitRepositoryInterface.cs

@@ -1,5 +1,6 @@
 
 using System;
+using System.IO;
 using ln.build.commands;
 using ln.http;
 
@@ -18,8 +19,15 @@ namespace ln.build.repositories
         public override void CloneSources(CIJob job)
         {
             job.Logger.Log("{0}: cloning repository to {1}", GetType().Name, job.WorkingDirectory);
-            bool success = (new CommandRunner("git", "clone", job.RepositoryURL, job.WorkingDirectory).Run(job.Environment) == 0) &&
+            job.Environment.WorkingDirectory = Path.GetTempPath();
-                    (!job.ContainsVariable("COMMIT_ID") || new CommandRunner("git", "checkout", job.GetVariable("COMMIT_ID")){ WorkingDirectory = job.WorkingDirectory, }.Run(job.Environment) == 0);
+
+            bool success = new CommandRunner("git", "clone", job.RepositoryURL, job.WorkingDirectory).Run(job.Environment) == 0;
+            job.Environment.WorkingDirectory = job.WorkingDirectory;
+
+            if (success && job.ContainsVariable("COMMIT_ID"))
+            {
+                success = new CommandRunner("git", "checkout", job.GetVariable("COMMIT_ID")).Run(job.Environment) == 0;
+            }
             if (!success)
                 throw new Exception("clone failed");
         }

ln.build/repositories/GiteaRepositoryInterface.cs

@@ -19,6 +19,13 @@ namespace ln.build.repositories
             BaseURL = baseURL;
         }
 
+        HttpClient CreateHttpClient()
+        {
+            HttpClient httpClient = new HttpClient();
+            httpClient.DefaultRequestHeaders.Add("Authorization",String.Format("token {0}", AuthorizationToken));
+            return httpClient;
+        }
+
         public override bool DetectValidity(string cloneUrl) => cloneUrl.StartsWith(BaseURL);
         public override void UpdateBuildState(CIJob job)
         {
@@ -37,10 +44,8 @@ namespace ln.build.repositories
                 stateObject.Add("state", job.BuildState.ToString().ToLower());
                 stateObject.Add("target_url", job.CIService.GetJobURL(job));
 
-                using (HttpClient httpClient = new HttpClient())
+                using (HttpClient httpClient = CreateHttpClient())
                 {
-                    httpClient.DefaultRequestHeaders.Add("Authorization",String.Format("token {0}", AuthorizationToken));
-
                     HttpResponseMessage response = httpClient.PostAsync(buildStateURL, new StringContent(stateObject.ToString(),Encoding.UTF8,"application/json")).Result;
                     job.Logger.Log(LogLevel.DEBUG, "UpdateBuildState({0}): {1}", job.BuildState, buildStateURL);
                     job.Logger.Log(LogLevel.DEBUG, "Request: {0}", stateObject.ToString());
@@ -82,11 +87,23 @@ namespace ln.build.repositories
                                 .SetVariable("NUGET_SOURCE", "http://nuget.l--n.de/nuget/l--n/v3/index.json")
                                 ;
 
-                            job.UpdateBuildState(BuildState.PENDING);
+                            using (HttpClient httpClient = CreateHttpClient())
+                            {
+                                string triggerFile = string.Format("{3}/api/v1/repos/{0}/{1}/contents/build.ln?ref={2}",
+                                                            job.GetVariable("REPO_OWNER"),
+                                                            job.GetVariable("REPO_NAME"),
+                                                            job.GetVariable("COMMIT_ID"),
+                                                            BaseURL
+                                                            );
 
-                            ciService.Enqueue(job);
+                                HttpResponseMessage triggerResponse = httpClient.GetAsync(triggerFile).Result;
+                                if (triggerResponse.StatusCode == System.Net.HttpStatusCode.OK)
+                                {
+                                    job.UpdateBuildState(BuildState.PENDING);
+                                    ciService.Enqueue(job);
+                                }
+                            }
                         }
-
                     } catch (Exception e)
                     {
                         response.StatusCode = 500;