ln.skyscanner/crawl/service/HTTP.cs

// /**
//  * File: HTTP.cs
//  * Author: haraldwolff
//  *
//  * This file and it's content is copyrighted by the Author and / or copyright holder.
//  * Any use wihtout proper permission is illegal and may lead to legal actions.
//  *
//  *
//  **/
using System;
using System.Net.Http;
using ln.logging;
using ln.types.net;
using System.Linq;
using System.Net;
using System.IO;
namespace ln.skyscanner.crawl.service
{
    public class HTTP : CrawlService
    {
        public HTTP()
            :base("http")
        {
        }

        public override bool Check(Crawl crawl)
        {
            foreach (int port in new int[] { 13080, 80 })
            {
                if (crawl.CheckRequiredOption("tcp", port))
                {
                    if (CheckPort(crawl, port))
                        return true;
                }
            }
            return false;
        }

        private bool CheckPort(Crawl crawl,int port)
        {
            foreach (IPv4 ip in crawl.Host.IPAddresses.Concat(new IPv4[] { crawl.Host.PrimaryIP }).Distinct())
            {
                try
                {
                    crawl.setState("Check HTTP at {0}:{1}",ip,port);

                    HttpWebRequest webRequest = WebRequest.CreateHttp(String.Format("http://{0}:{1}", ip, port));
                    webRequest.AllowAutoRedirect = true;
                    HttpWebResponse response = (HttpWebResponse)webRequest.GetResponse();

                    String content = null;

                    using (StreamReader reader = new StreamReader(response.GetResponseStream()))
                    {
                        content = reader.ReadToEnd();
                        reader.Close();
                    }

                    crawl.Host.SetHint("http.ip", ip);
                    crawl.Host.SetHint("http.port", port);
                    crawl.Host.SetHint("http.status", response.StatusCode);
                    crawl.Host.SetHint("http.server", response.Server);
                    crawl.Host.SetHint("http.content", content);


                    return true;
                }
                catch (WebException)
                {
                }
            }
            return false;
        }


        public override bool HostProvidesOption(Crawl crawl, params object[] parameters)
        {
            if (parameters.Length == 0)
                return false;

            foreach (object oport in parameters)
            {
                int port = (int)oport;
                if (!crawl.Host.GetHint(String.Format("http.{0}", port), false))
                    return false;
            }
            return true;
        }

    }
}