Scikit-learn based Data Analysis System (Programmable Web Crawler)

from flask import Flask, url_for, request, redirect, render_template

#
# Classification
#
@app.route('/classification_results', methods = ['POST'])
def classification_results():
    classifier = request.form['model']
    data_model= request.form['data']
    file = request.form['file']
    
    pg = program.Main(classifier, data_model, file)
    return 'Successful!'
    

# Combine template and method
@app.route('/classification/')
def classification(name=None):
    return render_template('classfication.html')

    ...
    ...
    ...
if __name__ == '__main__':
   app.run(debug = True)

        cat, des = [], []
        with open(file_name) as txtData:    
            lines = txtData.readlines()
            for line in lines:
                line = line.strip().split('$#$') 
                # keep the description
                des.append(self.remove_punctuation(line[7]))
                # keep the tag
                cat.append(line[18])
        df = pd.DataFrame(data = {'cat': cat, 'dess': des})
        
        #drop na
        df.dropna(axis=0, how='any', inplace=True)
        df[df.isnull().values==True]
        df = df[pd.notnull(df['dess'])]
        df = df.reset_index(drop=True)

def preprocess_complete(text):
    """
        words lemmatization: verb passed tense to current tense
        @text: a string
        return: a string
    """
    result = ''
    for token in gensim.utils.simple_preprocess(text):
        if token not in gensim.parsing.preprocessing.STOPWORDS and len(token) > 3:
            result= result + lemmatize_stemming(token) + ' '
    return result

def lemmatize_stemming(text):  
    """
        Words stemming: cats -> cat; meeting -> meet
    """
    stemmer = SnowballStemmer('english')
    return stemmer.stem(WordNetLemmatizer().lemmatize(text, pos='v'))

services/service - > servic
Restful/RESTful/REST/rest/... - > rest

X_train, X_test, y_train, y_test = train_test_split(df['des'], df['cat_id'], random_state = 0,stratify=df['cat_id'])

    count_vect = CountVectorizer()
    tfidf_transformer = TfidfTransformer()

    # transform dataset into vectors
    X_train_counts = count_vect.fit_transform(X_train)

    # transform vectors into weighted vectors 
    X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)

Results:
[[0. 0. 0. ... 0. 0. 0.]
 [0. 0. 0. ... 0. 0. 0.]
 [0. 0. 0. ... 0. 0. 0.]
 ...
 [0. 0. 0. ... 0. 0. 0.]
 [0. 0. 0. ... 0. 0. 0.]
 [0. 0. 0. ... 0. 0. 0.]] 

    count_vect = CountVectorizer()
    X_train_counts = count_vect.fit_transform(X_train)

    # use the trained doc vectors transform the test dataset into the vectors
    X_test_counts =  count_vect.transform(X_test)

    # initialize a LDA model
    # # of topics: 6
    lda_model = LatentDirichletAllocation(n_components=len(cat_id_df.values),
                                    learning_method='online',
                                    max_iter=20,
                                    random_state=42)
    
    # Use the LDA instance to model the dataset into the vectors of topics
    Xtopics = lda_model.fit_transform(X_train_counts)

    # transform the test doc into the vectors of topics
    Xtopics_test = lda_model.transform(X_test_counts)

Results:
[[0.00416773 0.00417269 0.00422381 0.00417282 0.00416738 0.97909557]
 [0.00420153 0.12530752 0.36135089 0.00418112 0.00418709 0.50077185]
 [0.00308746 0.00311789 0.49922885 0.00308779 0.00309947 0.48837853]
 ...
 [0.17848864 0.00416919 0.77270219 0.03626544 0.00417198 0.00420257]
 [0.00245217 0.00245509 0.80951414 0.00245443 0.00245213 0.18067205]
 [0.0053782  0.27290006 0.70556677 0.00537981 0.00537642 0.00539873]]

    train_tagged = train.apply(
            lambda r: TaggedDocument(words=r['des'], tags=[r['cat']]), axis=1)
    test_tagged = test.apply(
            lambda r: TaggedDocument(words=r['des'], tags=[r['cat']]), axis=1)

    cores = multiprocessing.cpu_count()
    model_dbow = Doc2Vec(vector_size = 5, window = 2, min_count = 1, workers=cores)
    model_dbow.build_vocab([x for x in tqdm(train_tagged.values)])

    # training the neural network model 30 times for the better vectorerization accuracy
    for epoch in range(30):
        model_dbow.train(utils.shuffle([x for x in tqdm(train_tagged.values)]), total_examples=len(train_tagged.values), epochs=1)
        model_dbow.alpha -= 0.002
        model_dbow.min_alpha = model_dbow.alpha

    y_train, X_train = vec_for_learning(model_dbow, train_tagged)
    y_test, X_test = vec_for_learning(model_dbow, test_tagged)

def vec_for_learning(model, tagged_docs):
    sents = tagged_docs.values
    # transform the tagged docs into the vectors paragraph vectors
    targets, regressors = zip(*[(doc.tags[0], model.infer_vector(doc.words, steps=20)) for doc in sents])
    return targets, regressors

# Results for tagged document:
TaggedDocument(['metro', 'publish', 'cloud', 'manag', 'build', 'help', 'media', 'compani', 'succeed', 'onlin', 'grow', 'readership', 'increas', 'revenu', 'build', 'brand', 'metro', 'publish', 'expos', 'rest', 'authent', 'base', 'oauth', 'specif', 'respons', 'json', 'format'], ['Enterprise'])
TaggedDocument(['pastebincom', 'websit', 'user', 'store', 'text', 'certain', 'period', 'time', 'site', 'programm', 'store', 'piec', 'sourc', 'code', 'configur', 'inform', 'kind', 'text', 'store', 'offer', 'function', 'site', 'includ', 'creat', 'list', 'delet', 'past', 'get', 'user', 'inform', 'set', 'use', 'rest', 'call', 'respons', 'format', 'document', 'requir', 'registr'], ['Tools'])
TaggedDocument(['coderwal', 'communiti', 'base', 'social', 'network', 'creat', 'purpos', 'allow', 'user', 'creat', 'lightweight', 'profil', 'display', 'achiev', 'cod', 'develop', 'user', 'gain', 'public', 'display', 'badg', 'trex', 'achiev', 'ampquothav', 'origin', 'repo', 'domin', 'languageampquot', 'octopusi', 'achiev', 'ampquothav', 'repo', 'follow', 'member', 'github', 'teamampquot', 'user', 'join', 'creat', 'compani', 'team', 'compet', 'highest', 'score', 'coderwal', 'profil', 'simpli', 'expos', 'json', 'represent', 'profil', 'retriev', 'http', 'valid', 'usernam', 'includ', 'request', 'respons', 'consist', 'profil', 'inform', 'list', 'achiev', 'return', 'json', 'jsonp'], ['Social'])
TaggedDocument(['circleci', 'platform', 'develop', 'creat', 'test', 'integr', 'deploy', 'code', 'project', 'circleci', 'offer', 'featur', 'like', 'push', 'code', 'github', 'deploy', 'code', 'server', 'circleci', 'allow', 'develop', 'access', 'integr', 'function', 'circleci', 'applic', 'exampl', 'method', 'includ', 'retriev', 'user', 'inform', 'retriev', 'project', 'inform', 'retriev', 'build', 'project'], ['Tools'])

        clf = tree.DecisionTreeClassifier()
        clf = clf.fit(X_trained, y_train)
        y_pred = clf.predict(X_test)
        print('Testing accuracy %s' % accuracy_score(y_test, y_pred))
        print('Testing F1 score: {}'.format(f1_score(y_test, y_pred, average='weighted')))

    clf = MultinomialNB()
    clf = clf.fit(X_trained, y_train)
    y_pred = clf.predict(X_trained)
    print('Testing accuracy %s' % accuracy_score(y_test, y_pred))
    print('Testing F1 score: {}'.format(f1_score(y_test, y_pred, average='weighted')))

    knn = KNeighborsClassifier()
    knn = knn.fit(X_trained, y_train)
    y_pred = knn.predict(X_trained)
    print('Testing accuracy %s' % accuracy_score(y_test, y_pred))
    print('Testing F1 score: {}'.format(f1_score(y_test, y_pred, average='weighted')))

    # sets K equals to 6
    clf = KMeans(n_clusters=len(cat_id_df.values))
    s = clf.fit(X_train)
    labels = s.labels_
    print(labels)
    # using Silhouette distance to evaluate the clustering
    print(silhouette_score(X_train, labels))

    clf = skc.DBSCAN(eps=0.3, min_samples=10)
    s = clf.fit(X_train)
    labels = s.labels_
    print(labels)
    print(silhouette_score(X_train, labels))