Getting all a hrefs from a webpage -- F# warm-ups

Been needing this for quite some time, and decided it was about time to tackle it as a way to flex my F# muscles a bit. I wanted to make this a command line utility, but haven't gotten around to it yet. If I do, I'll have it consume the html from STDIN, and spit the links to STDOUT. I want to do it that way to allow flexibility in how the pages are fetched.

Here is the code:

	open System.IO
	// from: http://htmlagilitypack.codeplex.com/
	#r @"C:\local\lib\HtmlAgilityPack\HtmlAgilityPack.dll"
	open HtmlAgilityPack
	let getLinks url =
	let fetchdata (url:string) =
	let req = System.Net.WebRequest.Create(url)
	let resp = req.GetResponse()
	let stream = resp.GetResponseStream()
	let reader = new StreamReader(stream)
	let data = reader.ReadToEnd()
	resp.Close()
	data
	let doc = new HtmlDocument()
	doc.LoadHtml (fetchdata url)
	let links = doc.DocumentNode.SelectNodes "//a[@href]"
	links
	|> Seq.map (fun x -> x.GetAttributeValue ("href", "no url"), x)
	|> Seq.filter (fun (x,y) -> x.StartsWith "http")
	|> Seq.distinctBy (fun (x,y) -> x)
	|> Seq.iter (fun (x,y) -> (printfn "%s [%s]" x y.InnerText))

view raw getLinks.fsx hosted with ❤ by GitHub