Introduce CommandEnvironment and CommandRunner.Argument

ln.build.server/Program.cs

@@ -42,7 +42,10 @@ namespace ln.build.server
                                 .SetVariable("REPO_OWNER", message["repository"]["owner"]["username"].ToNative().ToString())
                                 .SetVariable("REPO_NAME", message["repository"]["name"].ToNative().ToString())
                                 .SetVariable("COMMIT_ID", commitID)
-                                .SetVariable("NOTIFY", message["pusher"]["email"].ToNative().ToString());
+                                .SetVariable("NOTIFY", message["pusher"]["email"].ToNative().ToString())
+                                .SetVariable("NUGET_APIKEY", "3yAJPMxcaEhb_HP62dxK")
+                                .SetVariable("NUGET_SOURCE", "http://nuget.l--n.de/nuget/l--n/v3/index.json")
+                                ;
 
                             job.UpdateBuildState(BuildState.PENDING);
 

ln.build/CIJob.cs

@@ -7,9 +7,11 @@ using System.Reflection;
 using System.Reflection.Metadata.Ecma335;
 using System.Security.Cryptography.X509Certificates;
 using System.Text;
+using ln.build.commands;
 using ln.json;
 using ln.logging;
 using ln.threading;
+using Microsoft.VisualBasic;
 
 namespace ln.build
 {
@@ -18,16 +20,16 @@ namespace ln.build
         static HttpClient httpClient = new HttpClient();
 
         public string JobID { get; } = Guid.NewGuid().ToString("N");
-
         public string RepositoryURL { get; }
-
         public string WorkingDirectory { get; set; }
         public CIService CurrentCIService { get; set; }
 
         public Logger Logger { get; private set; }
 
+        public CommandEnvironment Environment { get; }
+
         List<PipeLine> pipeLines = new List<PipeLine>();
-        Dictionary<string,string> variables = new Dictionary<string, string>();
+        
 
         Dictionary<string,MemoryStream> logStreams = new Dictionary<string, MemoryStream>();
         Dictionary<string,Logger> logStreamLoggers = new Dictionary<string, Logger>();
@@ -39,6 +41,8 @@ namespace ln.build
             Logger.Backends.Add(Logger.ConsoleLogger);
 
             RepositoryURL = repositoryURL;
+
+            Environment = new CommandEnvironment();
         }
 
         public Stream GetLogStream(string name)
@@ -61,45 +65,10 @@ namespace ln.build
             return logStreamLogger;
         }
 
-        public string GetVariable(string varName) => GetVariable(varName, null);
-        public string GetVariable(string varName,string defValue)
-        {
-            if (!variables.TryGetValue(varName,out string value))
-                value = defValue;
-            return value;
-        }
-        public CIJob SetVariable(string varName,string value)
-        {
-            if (value != null)
-                variables[varName] = value;
-            else
-                variables.Remove(varName);
-
-            return this;
-        }
-        public void ExtendVariable(string varName,string value) => ExtendVariable(varName, value, ':');
-        public void ExtendVariable(string varName, string value, char seperator)
-        {
-            String currentValue = GetVariable(varName, "");
-            if (String.Empty.Equals(currentValue))
-            {
-                currentValue = value;
-            } else {
-                currentValue = string.Format("{0}{1}{2}",currentValue, seperator, value);
-            }
-            SetVariable(varName, currentValue);
-        }
-
-        public bool ContainsVariable(string varName) => variables.ContainsKey(varName);
-        public bool ContainsVariable(params string[] varNames)
-        {
-            foreach (string varName in varNames)
-            {
-                if (!variables.ContainsKey(varName))
-                    return false;
-            }
-            return true;
-        }
+        public string GetVariable(string varName) => Environment.Get(varName);
+        public string GetVariable(string varName,string defValue) => Environment.Get(varName, defValue);
+        public bool ContainsVariable(string varName) => Environment.Contains(varName);
+        public CIJob SetVariable(string varName,string varValue){ Environment.Set(varName,varValue); return this; }
 
         public async void UpdateBuildState(BuildState buildState)
         {
@@ -158,8 +127,8 @@ namespace ln.build
         {
             Logging.Log("cloning repository to {0}", WorkingDirectory);
             return 
-                (new CommandRunner(Logger, "git", "clone", RepositoryURL, WorkingDirectory).Run() == 0) &&
-                (!ContainsVariable("COMMIT_ID") || new CommandRunner(Logger, "git", "checkout", GetVariable("COMMIT_ID")){ WorkingDirectory = this.WorkingDirectory, }.Run() == 0);
+                (new CommandRunner(Logger, "git", "clone", RepositoryURL, WorkingDirectory).Run(Environment) == 0) &&
+                (!ContainsVariable("COMMIT_ID") || new CommandRunner(Logger, "git", "checkout", GetVariable("COMMIT_ID")){ WorkingDirectory = this.WorkingDirectory, }.Run(Environment) == 0);
         }
 
         public bool DetectPipelines()
@@ -177,11 +146,20 @@ namespace ln.build
 
         public void Notify()
         {
+            foreach (string key in logStreams.Keys)
+            {
+                logStreams[key].Position = 0;
+                string logContent;
+                using (StreamReader sr = new StreamReader(logStreams[key]))
+                    logContent = sr.ReadToEnd();
+
+                Logger.Log(LogLevel.INFO, "------------------- LogStream {0} ---------------------------\n{1}", key, logContent);
+            }
         }
 
         public void Cleanup()
         {
-            Directory.Delete(WorkingDirectory, true);
+            //Directory.Delete(WorkingDirectory, true);
         }
 
 

ln.build/CommandRunner.cs

@@ -1,132 +0,0 @@
-using System;
-using System.Diagnostics;
-using System.IO;
-using ln.logging;
-
-namespace ln.build
-{
-    public enum CRThrow { NEVER, NEGATIVE, NONZERO }
-    public class CommandRunner
-    {
-        public static CRThrow DefaultThrow { get; set; } = CRThrow.NONZERO;
-
-        public CRThrow Throw { get; set; } = DefaultThrow;
-        public string Executable { get; }
-        public string[] Arguments { get; set; }
-        public string WorkingDirectory { get; set; } = ".";
-
-        public Logger Logger { get; }
-
-        Func<int,bool> TestExitCode = null;
-
-        public CommandRunner(string executable,params string[] arguments) : this(Logger.Default, executable, arguments){}
-        public CommandRunner(Logger logger, string executable,params string[] arguments) : this(logger, null, executable, arguments){}
-        public CommandRunner(Logger logger, Func<int,bool> testExitCode, string executable,params string[] arguments)
-        {
-            Logger = logger;
-            Executable = executable;
-            Arguments = arguments;
-            
-            TestExitCode = testExitCode;
-        }
-
-        public string FindFileInPath(string filename)
-        {
-            Logger.Log(LogLevel.DEBUG, "Looking up {0} in paths {1}", filename, Environment.GetEnvironmentVariable("PATH"));
-
-            foreach (string path in Environment.GetEnvironmentVariable("PATH").Split(Path.PathSeparator))
-            {
-                string fullpath = Path.Combine(path,filename);
-                if (File.Exists(fullpath))
-                    return fullpath;
-            }
-
-            return filename;
-        }
-
-        public int Run() => Run(Logger, null, null);
-        public int Run(Stream stdout, Stream stderr) => Run(Logger, stdout, stderr);
-        public int Run(Logger logger, Stream stdout) => Run(logger, stdout, stdout);
-        public int Run(Logger logger, Stream stdout, Stream stderr)
-        {
-            ProcessStartInfo psi = new ProcessStartInfo(FindFileInPath(Executable), string.Join(' ', Arguments))
-            {
-                CreateNoWindow = true,
-                RedirectStandardError = true,
-                RedirectStandardOutput = true,
-                RedirectStandardInput = true,
-                WorkingDirectory = this.WorkingDirectory
-            };
-            psi.EnvironmentVariables.Remove("LANG");
-
-            logger.Log(LogLevel.INFO, "Executing: {0} {1}", psi.FileName, psi.Arguments);
-
-            Process process = Process.Start(psi);
-            process.WaitForExit();
-
-            if (stdout != null)
-                process.StandardOutput.BaseStream.CopyTo(stdout);
-            if (stderr != null)
-                process.StandardError.BaseStream.CopyTo(stderr);
-
-            logger.Log(LogLevel.INFO, "Result: {0}", process.ExitCode);
-
-            if (
-                ((TestExitCode != null) && !TestExitCode(process.ExitCode)) ||
-                ((Throw == CRThrow.NEGATIVE) && (process.ExitCode < 0)) ||
-                ((Throw == CRThrow.NONZERO) && (process.ExitCode != 0))
-            )
-                throw new Exception(String.Format("{0} execution gave result {1}", psi.FileName, process.ExitCode));
-
-            return process.ExitCode;
-        }
-
-        public int Run(out string stdout,out string stderr) => Run(Logger, out stdout, out stderr);
-        public int Run(Logger logger, out string stdout,out string stderr)
-        {
-            MemoryStream outstream,errstream;
-            
-            outstream = new MemoryStream();
-            errstream = new MemoryStream();
-
-            int status = Run(logger, outstream, errstream);
-
-            using (StreamReader sr = new StreamReader(outstream))
-                stdout = sr.ReadToEnd();
-
-            using (StreamReader sr = new StreamReader(errstream))
-                stderr = sr.ReadToEnd();
-
-            return status;
-        }
-
-
-
-
-
-
-
-/*
-        public static int Run(string executable,out Stream stdout,out Stream stderr,params string[] arguments)
-        {
-            CommandRunner runner = new CommandRunner(executable,arguments);
-            return runner.Run(out stdout,out stderr);
-        }
-
-        public static int Run(string executable,out string stdout,out string stderr,params string[] arguments) => Run(Logger.Default, executable, out stdout, out stderr, arguments);
-        public static int Run(Logger logger, string executable, out string stdout, out string stderr, params string[] arguments)
-        {
-            CommandRunner runner = new CommandRunner(logger, executable, arguments);
-            int result = runner.Run(out stdout, out stderr);
-
-            return result;
-        }
-
-        public static int Run(string executable,params string[] arguments) => Run(executable,out Stream stdout,out Stream stderr,arguments);
-        public static int Run(Logger logger, string executable, params string[] arguments){
-
-            return Run(executable,out Stream stdout,out Stream stderr,arguments);
-        }
-*/
-    }
-}

ln.build/DotNetPipeLine.cs

@@ -2,11 +2,13 @@
 using System;
 using System.IO;
 using System.Reflection.Metadata.Ecma335;
+using System.Runtime.InteropServices;
 using System.Runtime.InteropServices.WindowsRuntime;
 using System.Runtime.Serialization.Formatters;
 using System.Text;
 using System.Text.RegularExpressions;
 using LibGit2Sharp.Handlers;
+using ln.build.commands;
 using ln.logging;
 using ln.type;
 
@@ -60,7 +62,7 @@ namespace ln.build
             foreach (Match match in matches)
             {
                 Logging.Log(LogLevel.INFO,"filterPackageFilenames(): {0}", match.Groups[2].Value);
-                job.ExtendVariable("DOTNET_PACKAGES", match.Groups[2].Value);
+                job.Environment.Extend("DOTNET_PACKAGES", match.Groups[2].Value);
             }
         }
 
@@ -87,18 +89,25 @@ namespace ln.build
         }
         class PublishStep : CommandStep
         {
-            public PublishStep():base("push", "dotnet","nuget", "push", "<filename>", "-s", "<source>", "-k", "<apikey>"){ }
+            CommandRunner.Argument FileName { get; } = new CommandRunner.Argument(null);
+            CommandRunner.Option NugetApiKey { get; } = new CommandRunner.Option("-k", (e) => e.Get("NUGET_APIKEY"), true);
+            CommandRunner.Option NugetSource { get; } = new CommandRunner.Option("-s", (e) => e.Get("NUGET_SOURCE"));
+            public PublishStep():base("push", "dotnet","nuget", "push" )
+            {
+                CommandRunner.AddArguments(FileName, NugetApiKey, NugetSource);
+            }
 
             public override int Run(CIJob job)
             {
                 bool success = true;
 
-                CommandRunner.Arguments[4] = "http://nuget.l--n.de/nuget/l--n/v3/index.json";
-                CommandRunner.Arguments[6] = "3yAJPMxcaEhb_HP62dxK";
+//                CommandRunner.Arguments[4] = "http://nuget.l--n.de/nuget/l--n/v3/index.json";
+//                CommandRunner.Arguments[6] = "3yAJPMxcaEhb_HP62dxK";
 
                 foreach (string package in job.GetVariable("DOTNET_PACKAGES","").Split(':',StringSplitOptions.RemoveEmptyEntries))
                 {
-                    CommandRunner.Arguments[2] = package;
+                    FileName.SetValue(package);
+
                     if (base.Run(job) != 0)
                         success = false;
                 }

ln.build/PipeLine.cs

@@ -1,8 +1,7 @@
 
 using System;
 using System.Collections.Generic;
-using System.IO;
-using System.Text.Json.Serialization;
+using ln.build.commands;
 using ln.logging;
 
 namespace ln.build 
@@ -57,7 +56,7 @@ namespace ln.build
 
         public CommandRunner CommandRunner { get; }
 
-        public CommandStep(string stepName, string filename,params string[] arguments)
+        public CommandStep(string stepName, string filename,params CommandRunner.Argument[] arguments)
         :base(stepName)
         {
             CommandRunner = new CommandRunner(filename, arguments);
@@ -67,7 +66,7 @@ namespace ln.build
         {
             CommandRunner.WorkingDirectory = job.WorkingDirectory;
             
-            int result = CommandRunner.Run(job.Logger, job.GetLogStream(DefaultLogFileName));
+            int result = CommandRunner.Run(job.Environment, job.GetLogStream(DefaultLogFileName));
             OnCommandExited?.Invoke(this, job, result);
             return result;
         }

ln.build/commands/CommandEnvironment.cs

@@ -0,0 +1,134 @@
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using ln.logging;
+
+namespace ln.build.commands
+{
+    public class CommandEnvironment : IDictionary<string,string>
+    {
+        static string[] DefaultVariables = { "PATH", "HOME", "USERNAME" };
+        public Logger Logger { get; set; }
+        Dictionary<string,string> variables = new Dictionary<string, string>();
+
+        public ICollection<string> Keys => ((IDictionary<string, string>)variables).Keys;
+        public ICollection<string> Values => ((IDictionary<string, string>)variables).Values;
+        public int Count => ((ICollection<KeyValuePair<string, string>>)variables).Count;
+        public bool IsReadOnly => ((ICollection<KeyValuePair<string, string>>)variables).IsReadOnly;
+        public string this[string key] { get => Get(key); set => Set(key,value); }
+
+        public CommandEnvironment() : this(Logger.Default)
+        {
+            foreach (string defName in DefaultVariables)
+                Set(defName, Environment.GetEnvironmentVariable(defName));
+        }
+        public CommandEnvironment(Logger logger)
+        { 
+            Logger = logger;
+        }
+        public CommandEnvironment(IEnumerable<KeyValuePair<string,string>> setup) : this(Logger.Default, setup) { }
+        public CommandEnvironment(Logger logger, IEnumerable<KeyValuePair<string,string>> setup) : this(logger)
+        {
+            foreach (KeyValuePair<string,string> kvp in setup)
+                variables.Add(kvp.Key,kvp.Value);
+        }
+
+
+        public string Get(string varName) => Get(varName, null);
+        public string Get(string varName,string defValue)
+        {
+            if (!variables.TryGetValue(varName,out string value))
+                value = defValue;
+            return value;
+        }
+        public CommandEnvironment Set(string varName,string value)
+        {
+            if (value != null)
+                variables[varName] = value;
+            else
+                variables.Remove(varName);
+
+            return this;
+        }
+        public void Extend(string varName,string value) => Extend(varName, value, ':');
+        public void Extend(string varName, string value, char seperator)
+        {
+            string currentValue = Get(varName, "");
+            if (string.Empty.Equals(currentValue))
+            {
+                currentValue = value;
+            } else {
+                currentValue = string.Format("{0}{1}{2}",currentValue, seperator, value);
+            }
+            Set(varName, currentValue);
+        }
+
+        public bool Contains(string varName) => variables.ContainsKey(varName);
+        public bool Contains(params string[] varNames)
+        {
+            foreach (string varName in varNames)
+            {
+                if (!variables.ContainsKey(varName))
+                    return false;
+            }
+            return true;
+        }
+
+        public void Add(string key, string value)
+        {
+            ((IDictionary<string, string>)variables).Add(key, value);
+        }
+
+        public bool ContainsKey(string key)
+        {
+            return ((IDictionary<string, string>)variables).ContainsKey(key);
+        }
+
+        public bool Remove(string key)
+        {
+            return ((IDictionary<string, string>)variables).Remove(key);
+        }
+
+        public bool TryGetValue(string key, [MaybeNullWhen(false)] out string value)
+        {
+            return ((IDictionary<string, string>)variables).TryGetValue(key, out value);
+        }
+
+        public void Add(KeyValuePair<string, string> item)
+        {
+            ((ICollection<KeyValuePair<string, string>>)variables).Add(item);
+        }
+
+        public void Clear()
+        {
+            ((ICollection<KeyValuePair<string, string>>)variables).Clear();
+        }
+
+        public bool Contains(KeyValuePair<string, string> item)
+        {
+            return ((ICollection<KeyValuePair<string, string>>)variables).Contains(item);
+        }
+
+        public void CopyTo(KeyValuePair<string, string>[] array, int arrayIndex)
+        {
+            ((ICollection<KeyValuePair<string, string>>)variables).CopyTo(array, arrayIndex);
+        }
+
+        public bool Remove(KeyValuePair<string, string> item)
+        {
+            return ((ICollection<KeyValuePair<string, string>>)variables).Remove(item);
+        }
+
+        public IEnumerator<KeyValuePair<string, string>> GetEnumerator()
+        {
+            return ((IEnumerable<KeyValuePair<string, string>>)variables).GetEnumerator();
+        }
+
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return ((IEnumerable)variables).GetEnumerator();
+        }
+    }
+}

ln.build/commands/CommandRunner.cs

@@ -0,0 +1,171 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using ln.logging;
+
+namespace ln.build.commands
+{
+    public enum CRThrow { NEVER, NEGATIVE, NONZERO }
+    public class CommandRunner
+    {
+        public static CRThrow DefaultThrow { get; set; } = CRThrow.NONZERO;
+
+        public CRThrow Throw { get; set; } = DefaultThrow;
+        public string Executable { get; }
+
+        List<Argument> arguments = new List<Argument>();
+        public IEnumerable<Argument> Arguments => arguments;
+
+        public string WorkingDirectory { get; set; } = ".";
+
+        public Logger Logger { get; }
+
+        Func<int,bool> TestExitCode = null;
+
+        public CommandRunner(string executable,params Argument[] arguments) : this(Logger.Default, executable, arguments){}
+        public CommandRunner(Logger logger, string executable,params Argument[] arguments) : this(logger, null, executable, arguments){}
+        public CommandRunner(Logger logger, Func<int,bool> testExitCode, string executable,params Argument[] args)
+        {
+            Logger = logger;
+            Executable = executable;
+            arguments.AddRange(args);
+            
+            TestExitCode = testExitCode;
+        }
+
+        public void AddArgument(Argument argument) => arguments.Add(argument);
+        public void AddArguments(params Argument[] args) => arguments.AddRange(args);
+
+        public string FindFileInPath(CommandEnvironment environment, string filename)
+        {
+            Logger.Log(LogLevel.DEBUG, "Looking up {0} in paths {1}", filename, environment.Get("PATH",""));
+
+            foreach (string path in environment.Get("PATH","").Split(Path.PathSeparator))
+            {
+                string fullpath = Path.Combine(path,filename);
+                if (File.Exists(fullpath))
+                    return fullpath;
+            }
+
+            return filename;
+        }
+
+        IEnumerable<string> GetArguments(CommandEnvironment environment, bool mask) => Arguments.SelectMany((e)=>e.GetArguments(environment, mask));
+
+
+        public int Run() => Run(new CommandEnvironment(), null, null);
+        public int Run(CommandEnvironment environment) => Run(environment, null, null);
+        public int Run(CommandEnvironment environment, Stream stdout) => Run(environment, stdout, stdout);
+        public int Run(Stream stdout, Stream stderr) => Run(new CommandEnvironment(), stdout, stderr);
+        public int Run(CommandEnvironment environment, Stream stdout, Stream stderr)
+        {
+            ProcessStartInfo psi = new ProcessStartInfo(FindFileInPath(environment, Executable), string.Join(' ', GetArguments(environment, false)))
+            {
+                CreateNoWindow = true,
+                RedirectStandardError = true,
+                RedirectStandardOutput = true,
+                RedirectStandardInput = true,
+                WorkingDirectory = this.WorkingDirectory,
+            };
+
+            psi.Environment.Clear();
+            foreach (KeyValuePair<string,string> ev in environment)
+                psi.Environment.Add(ev.Key,ev.Value);
+
+            environment.Logger.Log(LogLevel.INFO, "Executing: {0} {1}", psi.FileName, string.Join(' ', GetArguments(environment, true)));
+
+            Process process = Process.Start(psi);
+            process.WaitForExit();
+
+            if (stdout != null)
+                process.StandardOutput.BaseStream.CopyTo(stdout);
+            if (stderr != null)
+                process.StandardError.BaseStream.CopyTo(stderr);
+
+            environment.Logger.Log(LogLevel.INFO, "Result: {0}", process.ExitCode);
+
+            if (
+                ((TestExitCode != null) && !TestExitCode(process.ExitCode)) ||
+                ((Throw == CRThrow.NEGATIVE) && (process.ExitCode < 0)) ||
+                ((Throw == CRThrow.NONZERO) && (process.ExitCode != 0))
+            )
+                throw new Exception(String.Format("{0} execution gave result {1}", psi.FileName, process.ExitCode));
+
+            return process.ExitCode;
+        }
+
+        public int Run(out string stdout,out string stderr) => Run(new CommandEnvironment(), out stdout, out stderr);
+        public int Run(CommandEnvironment environment, out string stdout,out string stderr)
+        {
+            MemoryStream outstream,errstream;
+            
+            outstream = new MemoryStream();
+            errstream = new MemoryStream();
+
+            int status = Run(environment, outstream, errstream);
+
+            using (StreamReader sr = new StreamReader(outstream))
+                stdout = sr.ReadToEnd();
+
+            using (StreamReader sr = new StreamReader(errstream))
+                stderr = sr.ReadToEnd();
+
+            return status;
+        }
+
+        
+        public class Argument
+        {
+            string value;
+            public bool MaskValue { get; set; }
+
+            protected Argument(){ }
+            public Argument(string argument) : this(argument, false) { }
+            public Argument(string argument, bool maskValue)
+            {
+                value = argument;
+                MaskValue = maskValue;
+            }
+
+            public void SetValue(string argValue) => value = argValue;
+
+            public virtual string[] GetArguments(CommandEnvironment environment, bool mask) => (value != null) ? new string[]{ (mask && MaskValue) ? "*****" : value } : new string[0];
+
+            public static implicit operator Argument(String s) => new Argument(s);
+        }
+
+        public class Option : Argument
+        {
+            public string OptionArgument { get; }
+            public Func<CommandEnvironment,string> GetValue { get; }
+
+            public Option(string optionArgument, string optionValue) : this(optionArgument, optionValue, false){ }
+            public Option(string optionArgument, string optionValue, bool maskValue)
+            {
+                OptionArgument = optionArgument;
+                GetValue = (e) => optionValue;
+                MaskValue = maskValue;
+            }
+            public Option(string optionArgument, Func<CommandEnvironment,string> getOptionValue) : this(optionArgument, getOptionValue, false) { }
+            public Option(string optionArgument, Func<CommandEnvironment,string> getOptionValue, bool maskValue)
+            {
+                OptionArgument = optionArgument;
+                GetValue = getOptionValue;
+                MaskValue = maskValue;
+            }
+
+            public override string[] GetArguments(CommandEnvironment environment, bool mask)
+            {
+                string value = GetValue(environment);
+                if ((value == null) || String.Empty.Equals(value))
+                {
+                    return new string[0];
+                } else {
+                    return new string[]{ OptionArgument, (mask && MaskValue) ? "*****" : value };
+                }
+            }
+        }
+    }
+}