using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Text.RegularExpressions;

public static class HTMLParser
{
    public static string Link(this string s, string url)
    {
        return string.Format("<a href=\"{0}\" target=\"_blank\">{1}</a>", url, s);
    }
    public static string ParseURL(this string s)
    {
        return Regex.Replace(s, @"(http(s)?://)?([\w-]+\.)+[\w-]+(/\S\w[\w- ;,./?%&=]\S*)?", new MatchEvaluator(HTMLParser.URL));
    }
    public static string ParseUsername(this string s)
    {
        return Regex.Replace(s, "(@)((?:[A-Za-z0-9-_]*))", new MatchEvaluator(HTMLParser.Username));
    }
    public static string ParseHashtag(this string s)
    {
        return Regex.Replace(s, "(#)((?:[A-Za-z0-9-_]*))", new MatchEvaluator(HTMLParser.Hashtag));
    }
    private static string Hashtag(Match m)
    {
        string x = m.ToString();
        string tag = x.Replace("#", "%23");
        return x.Link("http://search.twitter.com/search?q=" + tag);
    }
    private static string Username(Match m)
    {
        string x = m.ToString();
        string username = x.Replace("@", "");
        return x.Link("http://twitter.com/" + username);
    }
    private static string URL(Match m)
    {
        string x = m.ToString();
        return x.Link(x);
    }
}

So as you can see I’m using the new Extension Methods feature in C# 3.0.

Now I can simply just call the extension methods like this:

string tweet = "Just blogged about how to parse HTML from the @twitter timeline - http://jesal.us/blog/?p=132 #programming";
Response.Write(tweet.ParseURL().ParseUsername().ParseHashtag());

and the result should looks something like this:

Just blogged about how to parse html from the @twitter timeline – http://jesal.us/blog/?p=132 #programming

Just be sure to call ParseURL method before ParseUsername and ParseHashtag. The other two methods will add URLs to the usernames and hastags and you don’t want ParseURL to confuse those links with the original links present in the text.

This was inspired by Simon Whatley’s post about doing something similar using prototyping with JavaScript.

Here’s a very simple example of what you could do with it – I’m just extracting inner HTML from any element inside a HTML file which has a css class called “scrape” assigned to it:

using HtmlAgilityPack;

public partial class _Default : System.Web.UI.Page
{
    protected void Page_Load(object sender, EventArgs e)
    {
        HtmlDocument doc = new HtmlDocument();
        doc.Load(Server.MapPath(filePath));
        Parse(doc.DocumentNode);
    }
    private void Parse(HtmlNode n)
    {
        foreach (HtmlAttribute atr in n.Attributes)
        {
            if (atr.Name == "class" && atr.Value == "scrape")
            {
                Response.Write(n.InnerHtml);
            }
        }

        if (n.HasChildNodes)
        {
            foreach (HtmlNode cn in n.ChildNodes)
            {
                Parse(cn);
            }
        }
    }
}

That’s just a very small part of what it could do. I’ll expand upon this and post a few more examples in the future showing some interesting things you could do with this.

protected ArrayList getURL(string txtIn)
{
    ArrayList outURL = new ArrayList();
    Regex r = new Regex("href\\s*=\\s*(?:(?:\\\"(?<url>[^\\\"]*)\\\")|(?<url>[^\\s]* ))");
    MatchCollection mc1 = r.Matches(txtIn);

    foreach (Match m1 in mc1)
    {
        foreach (Group g in m1.Groups)
        {
            outURL.Add(g.Value);
        }
    }

    return outURL;
}

protected string StripSpecChars(string txtIn)
{
    string txtOut = Regex.Replace(txtIn, @"[^\w\.@-]", "").Trim();
    return txtOut;
}

protected string removeCommonWords(string sourceStr)
{
    string[] seperator = { " " };
    string[] ignoreWords = { "a", "all", "am", "an", "and", "any", "are", "as",
        "at", "be", "but", "can", "did", "do", "does", "for", "from", "had",
        "has", "have", "here", "how", "i", "if", "in", "is", "it", "no", "not",
        "of", "on", "or", "so", "that", "the", "then", "there", "this", "to",
        "too", "up", "use", "what", "when", "where", "who", "why", "you" };
    string[] outputStr = { };

    outputStr = sourceStr.ToLower().Split(seperator, StringSplitOptions.RemoveEmptyEntries);

    foreach (string unwantedWord in ignoreWords)
    {
        int index = Array.IndexOf(outputStr, unwantedWord);

        if (index != -1)
        {
            string[] copyStrArr = new string[outputStr.Length - 1];

            // copy the elements before the found index
            for (int i = 0; i < index; i++)
            {
                copyStrArr[i] = outputStr[i];
            }

            // copy the elements after the found index
            for (int i = index; i < copyStrArr.Length; i++)
            {
                copyStrArr[i] = outputStr[i + 1];
            }

            outputStr = copyStrArr;
        }
    }

    sourceStr = string.Join(" ", outputStr);

    return sourceStr;
}

Let me know if you guys have a better solution.

public static string TruncateText(string txtIn, int newLength)
{
    string txtOut = txtIn;
    string pattern = @"<(.|\n)*?>";

    //Strip out HTML tags
    if (Regex.IsMatch(txtIn, pattern, RegexOptions.None))
        txtOut = Regex.Replace(txtIn, pattern, string.Empty, RegexOptions.Multiline).Trim();

    if (txtOut.Length > newLength)
    {
        int endPos = txtOut.LastIndexOf(" ", newLength);
        txtOut = txtOut.Substring(0, endPos) + "...";
    }

    return txtOut;
}

I recently compiled a small list of sites/blogs which have excellent reference material on regular expressions. Here it is:

• The absolute bare minimum every programmer should know about regular expressions
• Regex Tutorial
• Regular Expressions Cheat Sheet
• regular-expressions.info
• Learn Regular Expression (Regex) syntax with C# and .NET

Hope that helps!!