Analysing Crawl Heritrix reports
Python Libraries used
In [1]:
%matplotlib inline
from pandas import DataFrame, read_csv
import matplotlib as plt
import matplotlib.pyplot as gplt
import pandas as pd
import sys
In [2]:
print 'Python version ' + sys.version
print 'Pandas version ' + pd.__version__
print 'Matplotlib version ' + plt.__version__
Heritrix Report: seeds-report.txt
In [3]:
dfseeds = read_csv('seeds-report.txt', sep = ' ')
In [5]:
dfseeds
Out[5]:
Seeds Return Codes Distribution
In [6]:
codes = dfseeds['[code]'].value_counts()
codes
Out[6]:
In [7]:
codes_to_plot = codes[:6]
In [19]:
plt = codes_to_plot.plot(kind='pie', figsize=(6, 6), autopct='%.2f%%',fontsize=15)
plt.set_title('Seeds return codes weigth', fontsize=25)
Out[19]:
Heritrix Report: hosts-report.txt
In [43]:
dfhosts = read_csv('hosts-report.txt', delimiter = ' ', index_col=False)
URL per site
In [73]:
dfhosts['TOTAL_URLS'] = dfhosts['[#urls]'] + dfhosts['[#remaining]']
order_dfhosts = dfhosts.ix[1:].sort(columns='TOTAL_URLS', ascending=False)
Out[73]:
In [81]:
plt = order_dfhosts[['TOTAL_URLS','[host]']][:20].plot(kind='bar',x='[host]',figsize=(10,10))
plt.set_ylabel('Number of URLs', fontsize=15)
plt.set_xlabel('Hosts', fontsize=15)
plt.set_title('TOP 20 Hosts with mode URLs', fontsize=25)
Out[81]:
Top 30 URL Crawled Hosts
In [85]:
dfhosts[['[#urls]','[host]']][1:31]
Out[85]:
Total URLS Remaining
In [13]:
dfhosts['[#remaining]'].sum()
Out[13]:
Top 20 hosts with URL remaining
In [87]:
ax = dfhosts[['[host]','[#remaining]']].sort(columns='[#remaining]', ascending = False)[:20].plot(kind='bar',x='[host]',figsize=(10,10))
ax.set_ylabel('Number of URLs', fontsize=15)
ax.set_xlabel('Hosts', fontsize=15)
ax.set_title('TOP 20 hosts with more URLs remaining', fontsize=25)
Out[87]:
Remaining URLs distribution
In [15]:
dfhosts['[#remaining]'].describe()
Out[15]:
Crawled URLs distirbution
In [19]:
dfhosts['[#urls]'].describe()
Out[19]:
Crawl logs Links Extracted from crawl.log
Round 1: cleaning seeds to just .eu uris
In [93]:
df_urls = read_csv('seeds_extracted.txt', header=None, names=['URL'])
df_eu_urls = df_urls[df_urls['URL'].str.contains('^.*.eu$')]
print "Number of seeds round1:"
df_eu_urls.count()
Out[93]:
Round 2: cleaning spam seeds indentified
In [99]:
df_eu_urls_2 = df_eu_urls[~df_eu_urls['URL'].str.contains('.*dbquanti.eu$|.*autobazar.eu$|.*in-links.eu$|.*myface4u.eu$|.*share-with.eu$|.*prace-jobs.eu$|.*cutegirls.eu$')]
print "Number of seeds round2:"
df_eu_urls_2.count()
Out[99]:
Round 3: Cleaning some more spam detected later (e-mp3s.eu)
In [100]:
df_eu_urls_3 = df_eu_urls_2[~df_eu_urls['URL'].str.contains('.*\.e-mp3s.eu$')]
print "Number of seeds round3:"
df_eu_urls_3.count()
Out[100]:
Top 20 Domains with more subdomains associated
In [109]:
d = DataFrame(df_eu_urls_3['URL'].apply(lambda str : str.split('.')[-2:]))
dauxfilter = DataFrame(d['URL'].apply(lambda x : x[0] + "." + x[1]))
dfilter = dauxfilter['URL'].value_counts()
dfilter[:20]
Out[109]: