Метка site-map

Скрипт для сканирования site-map

#!/usr/bin/env pythonimport sysimport refrom tld import get_tldimport linkGrabberdef get_links(url): links = linkGrabber.Links(url) for link in links.find(): curr_link = link['href'] curr_text = link['text'].encode('ascii', 'ignore').decode('ascii') if curr_link == url: continue if curr_link in scanned_links: continue if in_external_links(curr_link): continue if is_blacklisted(curr_link): continue if…