# -*- coding: utf-8 -*-

import urllib
import urlparse
import os
import re
import sys

class url_finder:

    def __init__(self, url):
        html = self.get_html(sys.argv[1])
        
        # url trimming
        url_parsed = urlparse.urlparse(url)
        self.url = '%s://%s' % (url_parsed[0], url_parsed[1])

        # exclude file name
        spl = url_parsed[2].split('/')
        if len(spl) > 1:
            self.url += '/'.join(spl[0:-1])

        # links, and images
        self.links = self.get_all_external_links(html)
        self.save_all_images(html)

    def get_all_external_links(self, html):
        found = re.findall(r'<a.*href=\"(https?.+?)\"', html)
        return found

    def save_all_images(self, html):
        if os.path.exists('./img') == False:
            os.mkdir('./img')
        
        for s in self.get_all_image_links(html):
            url = self.trim_image_url(s)
            filename = './img/' + s.split('/')[-1].strip()
            content = self.get_html(url)
            with open(filename, 'wb') as f:
                print "Saving [%s] to [%s]..." % (url, filename)
                f.write(content)

    def get_all_image_links(self, html):
        found = re.findall(r'<img.*src=\"(.+\.(gif|png|jpg))\"', html)
        return set(x[0] for x in found)
        
    def trim_image_url(self, src_url):
        if src_url[:4] == 'http':
            return src_url
        elif src_url[0] == '/':
            return "%s%s" % (self.url, src_url)
        else:
            return "%s/%s" % (self.url, src_url)

    def get_html(self, url):
        u = urllib.urlopen(url)
        html = u.read()
        u.close()
        return html

if __name__ == "__main__" :
    finder = url_finder("http://www.google.com")
    for x in finder.links:
        print x