Building a URL classifier using DMOZ data

import os
import sys
import re
import pandas as pd
import time
import numpy as np
import pickle
from collections import Counter

import keras
from keras import backend as K
from keras.utils.data_utils import get_file
from keras.utils import np_utils
from keras.utils.np_utils import to_categorical
from keras.models import Sequential, Model
from keras.layers import Input, Embedding, Reshape, merge, LSTM, Bidirectional
from keras.layers import TimeDistributed, Activation, SimpleRNN, GRU
from keras.layers.core import Flatten, Dense, Dropout, Lambda
from keras.regularizers import l2, activity_l2, l1, activity_l1
from keras.layers.normalization import BatchNormalization
from keras.optimizers import SGD, RMSprop, Adam
from keras.utils.layer_utils import layer_from_config
from keras.metrics import categorical_crossentropy, categorical_accuracy
from keras.layers.convolutional import *
from keras.preprocessing import image, sequence
from keras.preprocessing.text import Tokenizer

from sklearn import preprocessing, cross_validation, metrics, naive_bayes, pipeline, \
        feature_extraction
import operator
import matplotlib.pyplot as plt
import seaborn as sns

plt.style.use('ggplot')
%matplotlib inline

path = '/home/dcrush/Documents/Url Classifier'
fn = 'content.rdf.u8'

# initialise empty dataframe
df = pd.DataFrame(columns=['category', 'title', 'desc'])

# we will read everything in one go and then filter out useful stuff
with open(os.path.join(path, fn), 'r') as fl_in:
    lines = fl_in.readlines()

lines = [str(line) for line in lines]
titles = [re.findall('<d:Title>(.+)</d:Title>', line) for line in lines]
descs = [re.findall('<d:Description>(.+)</d:Description>', line) for line in lines]
topics = [re.findall('<topic>(.+)</topic>', line) for line in lines]
del lines

titles_pos = [i for i, x in enumerate(titles) if len(x)>0]
descs_pos = [i for i, x in enumerate(descs) if len(x)>0]
topics_pos = [i for i, x in enumerate(topics) if len(x)>0]

print '# titles found', len(titles_pos)
print '# descriptions found', len(descs_pos)
print '# topics found', len(topics_pos)

topics_list = []
titles_list = []
descs_list = []
for line_counter, i in enumerate(titles_pos):
    if line_counter % 1000000 == 0:
        print line_counter, 'processed'
    if len(descs[i+1])>0 and len(topics[i+2])>0:
        topics_list.append(topics[i+2][0].split('/')[1])
        titles_list.append(titles[i][0])
        descs_list.append(descs[i+1][0])

df.category = topics_list
df.title = titles_list
df.desc = descs_list

df.shape

df.head()

df.tail()

df = pd.read_csv(os.path.join(path, 'contents.csv'))

plt.figure(figsize=(12, 5))
df.category.value_counts().plot(kind='bar');
plt.title('Category counts');

print 'df shape before', df.shape
df = df[~df.category.isin(['World', 'Regional'])]
print 'df shape after', df.shape

plt.figure(figsize=(12, 5))
df.category.value_counts().plot(kind='bar');
plt.title('Category counts');

df.desc = df.title + ' ' + df.desc
df = df.drop(['title'], axis=1)
df.desc = df.desc.str.lower()

pipe = pipeline.Pipeline([('vect', feature_extraction.text.CountVectorizer()),
                         ('tfidf', feature_extraction.text.TfidfTransformer()),
                         ('gnb', naive_bayes.BernoulliNB())])
le = preprocessing.LabelEncoder()
le.fit(df.category)
y_data = le.transform(df.category)
X_data = df.desc.values
X_train, X_test, y_train, y_test = cross_validation.train_test_split(X_data, y_data, \
        test_size=0.2, random_state=1000)
pipe.fit(X_train, y_train)

print metrics.classification_report(pipe.predict(X_test), y_test, target_names=le.classes_)

lens = [len(x) for x in df.desc]
plt.figure(figsize=(12, 5));
print max(lens), min(lens), np.mean(lens)
sns.distplot(lens);
plt.title('Description length distribution');

vocab_size = 5000
seq_len = 200

words = [re.findall('[\w\d]+', x) for x in df.desc]

all_words = []
for x in words:
    all_words += x

word_to_id = Counter(all_words).most_common(vocab_size)

word_to_id[:10]

word_to_id[-10:]

word_to_id = {x[0]:i for i, x in enumerate(word_to_id)}

train = [np.array([word_to_id[y] if y in word_to_id else vocab_size-1\
        for y in x]) for x in words]

train = sequence.pad_sequences(train, maxlen=seq_len, value=0)

train = train.astype('float32')

y_data = to_categorical(y_data)
X_train, X_test, y_train, y_test = cross_validation.train_test_split(train, y_data, \
        test_size=0.2, random_state=1000)

model = Sequential([
        Embedding(vocab_size, 50, input_length=seq_len),
        Dropout(0.1),
        Convolution1D(64, 5, border_mode='same', activation='relu'),
        Dropout(0.2),
        Convolution1D(128, 5, border_mode='same', activation='relu'),
        Dropout(0.2),
        MaxPooling1D(),
        Flatten(),
        Dense(200, activation='relu'),
        Dropout(0.3),
        Dense(13, activation='softmax')])
model.compile(loss='categorical_crossentropy', optimizer=Adam(), \
        metrics=['accuracy'])

model.fit(X_train, y_train, validation_data=(X_test, y_test), \
        nb_epoch=3, batch_size=64)

model.optimizer.lr = 1e-4

model.fit(X_train, y_train, validation_data=(X_test, y_test), \
        nb_epoch=1, batch_size=64)

preds = [np.argmax(x) for x in model.predict(X_test)]
y_test_argmax = [np.argmax(x) for x in y_test]
print metrics.classification_report(preds, y_test_argmax, target_names=le.classes_)