The SSH host key has changed on 8 April, 2022 to this one: SHA256:573uTBSeh74kvOo0HJXi5ijdzRm8me27suzNEDlGyrQ
Simple web crawler written in Python
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

62 lines
1.4 KiB

7 years ago
import urllib2
import sys
from bs4 import BeautifulSoup
import requests
import re
def make_url(link, base_url):
if'^[a-z-]+:', link):
return link
return re.sub(r'([^:])//+', r'\1/', base_url + '/' + link)
def main():
checked_links = []
link_cache = []
if len(sys.argv) < 2:
print("Usage: %s <url>" % sys.argv[0])
return 1
base_url = sys.argv[1]
while True:
link = link_cache[0]
link_cache = [x for x in link_cache if x != link]
print("Checking %s" % link)
r = requests.get(link)
if r.status_code == 200:
soup = BeautifulSoup(r.content)
for a in soup.find_all('a'):
link = make_url(a.get('href'), base_url)
if link.startswith(base_url):
if link not in checked_links:
print("Skipping checked link %s" % link)
print("Skipping external link %s" % link)
print r.status_code
if len(link_cache) == 0:
print("Done. Checked links:")
with open('link_list.txt', 'w') as f:
for link in checked_links:
f.write(link + "\n")
print link
if __name__ == '__main__':