
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import re
import seaborn as sns
from sklearn import preprocessing
from sklearn.dummy import DummyClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neighbors import NearestNeighbors
from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import f1_score, recall_score, precision_score
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from sklearn.cluster import AgglomerativeClustering
from sklearn.cluster import DBSCAN
from sklearn.decomposition import PCA
from scipy.cluster.hierarchy import dendrogram, linkage
from sklearn.metrics import silhouette_score
from sklearn.metrics.pairwise import euclidean_distances

pd.options.mode.chained_assignment = None


title_basics = pd.read_csv("./datasets/title_basics.tsv", sep='\t', encoding="UTF-8")
title_ratings = pd.read_csv("./datasets/title_ratings.tsv", sep='\t', encoding="UTF-8")
title_crew = pd.read_csv("./datasets/title_crew.tsv", sep='\t', encoding="UTF-8")
name_basics = pd.read_csv("./datasets/name_basics.tsv", sep='\t', encoding="UTF-8")
title_principals = pd.read_csv("./datasets/title_principals.tsv", sep='\t', encoding="UTF-8")

C:\Users\franc\anaconda3\lib\site-packages\IPython\core\interactiveshell.py:3165: DtypeWarning: Columns (4,5) have mixed types.Specify dtype option on import or set low_memory=False.
  has_raised = await self.run_ast_nodes(code_ast.body, cell_name,


title_basics["runtimeMinutes"] = pd.to_numeric(title_basics["runtimeMinutes"], errors='coerce')
title_basics["genres"] = title_basics["genres"].astype(object)
title_basics["startYear"] = pd.to_numeric(title_basics["startYear"], errors='coerce')
title_basics["titleType"] = title_basics["titleType"].astype(object)
title_basics["isAdult"] = title_basics["isAdult"].astype(object)
title_ratings["averageRating"] = pd.to_numeric(title_ratings["averageRating"], errors='coerce')
title_ratings["numVotes"] = pd.to_numeric(title_ratings["numVotes"], errors='coerce')


title_basics.shape[0]

8246400


title_basics.isna().sum(axis=0)

tconst                  0
titleType               0
primaryTitle            9
originalTitle           9
isAdult                 0
startYear          971886
endYear                 0
runtimeMinutes    5920699
genres                 10
dtype: int64


only_movies = title_basics[(title_basics.genres != '\\N') & (title_basics.titleType == "movie")]


only_movies.shape[0]

516492


only_movies.isna().sum(axis=0)

tconst                 0
titleType              0
primaryTitle           0
originalTitle          0
isAdult                0
startYear          70672
endYear                0
runtimeMinutes    170153
genres                 0
dtype: int64


only_movies = only_movies.dropna()


only_movies.shape[0]

340916


only_movies.drop('endYear', axis=1, inplace=True)
only_movies.drop('titleType', axis=1, inplace=True)
only_movies.drop('originalTitle', axis=1, inplace=True)


movies = pd.merge(only_movies, title_ratings, on='tconst')


movies_2 = pd.merge(movies, title_crew, on='tconst')
movies_2


movDir = movies_2[['tconst', 'directors']]

# Se crea una lista de los directores
movDir["directors"] = movDir["directors"].str.split(",")

# Se separa cada uno de los directores por su película
md = movDir.explode("directors")

# Se mapea name_basics a la columna de directores
md["directors"] = md["directors"].map(name_basics.set_index("nconst").squeeze()['primaryName']).fillna("")

# Los nombres de los directores se agrupan para cada película
res = md.groupby("tconst").agg(",".join).reset_index()

# Se quitan las comas innecesarias de la nueva columna
res["directors"] = res["directors"].str.strip(",")

res


movies_3 = pd.merge(movies, res, on='tconst')
movies_3


movWri = movies_2[['tconst', 'writers']]

# Se crea una lista de los escritores
movWri["writers"] = movWri["writers"].str.split(",")

# Se separa cada uno de los escritores por su película
mw = movWri.explode("writers")

# Se mapea name_basics a la columna de escritores
mw["writers"] = mw["writers"].map(name_basics.set_index("nconst").squeeze()['primaryName']).fillna("")

# Los nombres de los escritores se agrupan para cada película
res2 = mw.groupby("tconst").agg(",".join).reset_index()

# Se quitan las comas innecesarias de la nueva columna
res2["writers"] = res2["writers"].str.strip(",")

res2


movies_4 = pd.merge(movies_3, res2, on='tconst')
movies_4


# Se seleccionan las columnas tconst y nconst, los identificadores de las películas y de cada persona, donde
# la persona asociada a cada nconst sea un actor o actriz.
actors_title = title_principals[title_principals['category'].isin(['actor', 'actress'])][['tconst', 'nconst']]

# Se mapea name_basics a la columna de nconst
actors_title['nconst'] = actors_title["nconst"].map(name_basics.set_index("nconst").squeeze()['primaryName']).fillna("")

# Se renombra la columna nconst como actors
actors_title = actors_title.rename(columns={'nconst':'actors'})

# Se consideran solo las películas que estén en la tabla creada anteriormente
actors_title = actors_title[actors_title['tconst'].isin(movies_4['tconst'])]

# Los nombres de los actores se agrupan para cada película
actors_title = actors_title.groupby("tconst").agg(",".join).reset_index()

# Se quitan las comas innecesarias
actors_title["actors"] = actors_title["actors"].str.strip(",")

actors_title


movies_5 = pd.merge(movies_4, actors_title, on='tconst')
movies_5.dropna()
movies_5


movies_5.isna().sum(axis=0)

tconst            0
primaryTitle      0
isAdult           0
startYear         0
runtimeMinutes    0
genres            0
averageRating     0
numVotes          0
directors         0
writers           0
actors            0
dtype: int64


movies_5.describe()


movies = movies_5
title_basics = None
title_ratings = None
title_crew = None
name_basics = None
title_principals = None

# Para exportar el dataset
#movies_5.to_csv("movies_final.tsv", sep='\t')


movies_alt = pd.read_csv("./datasets/movies_final.tsv", sep='\t')
movies_alt["Genero1"], movies_alt["Genero2"], movies_alt["Genero3"] = movies_alt["genres"].str.split(',', 2).str
movies_alt.drop('genres', axis=1, inplace=True)
movies_alt

<ipython-input-23-ac2338081543>:2: FutureWarning: Columnar iteration over characters will be deprecated in future releases.
  movies_alt["Genero1"], movies_alt["Genero2"], movies_alt["Genero3"] = movies_alt["genres"].str.split(',', 2).str


movies.sort_values(by="averageRating").quantile(0.9)

startYear         2018.0
runtimeMinutes     121.0
averageRating        7.5
numVotes          2481.0
Name: 0.9, dtype: float64


movies.sort_values(by="numVotes").quantile(0.9)

startYear         2018.0
runtimeMinutes     121.0
averageRating        7.5
numVotes          2481.0
Name: 0.9, dtype: float64


movies.sort_values(by="numVotes").quantile(0.1)

startYear         1951.0
runtimeMinutes      70.0
averageRating        4.3
numVotes            12.0
Name: 0.1, dtype: float64


best_movies = movies[(movies["averageRating"] >= 7.5) & (movies["numVotes"] >= 2481)]
worst_movies = movies[movies["averageRating"] <= 4.3]


round(best_movies.shape[0] / movies.shape[0] * 100, 2)

1.67


round(worst_movies.shape[0] / movies.shape[0] * 100, 2)

10.77


(best_movies[best_movies["isAdult"] == 1].shape[0], worst_movies[worst_movies["isAdult"] == 1].shape[0])

(0, 487)


best_movies1970 = best_movies[best_movies["startYear"] >= 1970]
best_movies1970.sort_values(by="averageRating", ascending=False).head(10)


worst_movies1970 = worst_movies[worst_movies["startYear"] >= 1970]
worst_movies1970.sort_values(by="averageRating").head(10)


# str -> double
# Entrega el porcentaje de películas del género que pertenecen a las mejores películas.
def best_movies_por_genero(genre):
    genero = best_movies[pd.Series(best_movies["genres"]).str.contains(genre).tolist()]
    return round(genero.shape[0] / best_movies.shape[0] * 100, 2)

# str -> double
# Entrega el porcentaje de películas del género que pertenecen a las peores películas
def worst_movies_por_genero(genre):
    genero = worst_movies[pd.Series(worst_movies["genres"]).str.contains(genre).tolist()]
    return round(genero.shape[0] / worst_movies.shape[0] * 100, 2)

# str -> boxplot
# Crea un boxplot de los rating de un género
def boxplot_rating_por_genero(genre):
    toplox = pd.DataFrame()
    genero = movies[pd.Series(movies["genres"]).str.contains(genre).tolist()]
    toplox['primaryTitle'] = genero['primaryTitle']
    toplox["0"] = (genero[genero["isAdult"] == 0])['averageRating']
    toplox["1"] = (genero[genero["isAdult"] == 1])['averageRating']
    (toplox[["0", "1"]]).plot(kind="box")
    plt.xlabel("¿Es para adultos? (0 = No, 1 = Sí)")
    plt.ylabel('Rating') 
    plt.title(f"Boxplot de ratings para género {genre}")
    plt.show()
    
# str -> boxplot
# Crea un boxplot con la distribución de ratings para una película con género principal genre
# Género principal es el primer género en la lista de géneros
def boxplot_rating_genero_principal(main_genre):
    toplox = pd.DataFrame()
    toplox['primaryTitle'] = movies_alt[(movies_alt["Genero1"] == main_genre)]['primaryTitle']
    toplox["0"] = (movies_alt[(movies_alt["Genero1"] == main_genre) & (movies_alt["isAdult"] == 0)])['averageRating']
    toplox["1"] = (movies_alt[(movies_alt["Genero1"] == main_genre) & (movies_alt["isAdult"] == 1)])['averageRating']
    (toplox[["0", "1"]]).plot(kind="box")
    plt.xlabel("¿Es para adultos? (0 = No, 1 = Sí)")
    plt.ylabel('Rating') 
    plt.title(f"Boxplot de ratings para género principal {main_genre}")
    plt.show()
    
# str -> plt
# Crea un gráfico de dispersión que muestra la relación entre rating y duración para películas de un género
def dispersion_rating_duracion(genre):
    toplox = pd.DataFrame()
    genero = movies[pd.Series(movies["genres"]).str.contains(genre).tolist()]
    toplox['primaryTitle'] = genero['primaryTitle']
    toplox["D"] = (movies[movies["genres"] == genre])['runtimeMinutes']
    toplox["R"] = (movies[movies["genres"] == genre])['averageRating']
    (toplox[["D", "R"]]).plot(kind="scatter", x="D", y="R")
    plt.xlabel("Duración")
    plt.ylabel('Rating') 
    plt.title(f"Dispersión entre rating y duración para el género {genre}")
    plt.xlim(0, 600)
    plt.show()
    
# str -> plt
# Crea un gráfico de dispersión que muestra la relación entre rating y cantidad de votos para películas de un género
def dispersion_rating_votos(genre):
    toplox = pd.DataFrame()
    genero = movies[pd.Series(movies["genres"]).str.contains(genre).tolist()]
    toplox['primaryTitle'] = genero['primaryTitle']
    toplox["V"] = (movies[movies["genres"] == genre])['numVotes']
    toplox["R"] = (movies[movies["genres"] == genre])['averageRating']
    (toplox[["V", "R"]]).plot(kind="scatter", x="V", y="R")
    plt.xlabel("Número de votos")
    plt.ylabel('Rating') 
    plt.title(f"Dispersión entre rating y votos para género {genre}")
    plt.show()


print(f'Action: {best_movies_por_genero("Action")}%')
print(f'Drama: {best_movies_por_genero("Drama")}%')
print(f'Crime: {best_movies_por_genero("Crime")}%')
print(f'Comedy: {best_movies_por_genero("Comedy")}%')
print(f'Romance: {best_movies_por_genero("Romance")}%')
print(f'Mystery: {best_movies_por_genero("Mystery")}%')
print(f'Family: {best_movies_por_genero("Family")}%')
print(f'Adult: {best_movies_por_genero("Adult")}%')
print(f'Sport: {best_movies_por_genero("Sport")}%')
print(f'Sci-Fi: {best_movies_por_genero("Sci-Fi")}%')

Action: 15.46%
Drama: 74.78%
Crime: 17.74%
Comedy: 27.93%
Romance: 18.97%
Mystery: 6.48%
Family: 3.97%
Adult: 0.0%
Sport: 1.87%
Sci-Fi: 2.79%


print(f'Action: {worst_movies_por_genero("Action")}%')
print(f'Drama: {worst_movies_por_genero("Drama")}%')
print(f'Crime: {worst_movies_por_genero("Crime")}%')
print(f'Comedy: {worst_movies_por_genero("Comedy")}%')
print(f'Romance: {worst_movies_por_genero("Romance")}%')
print(f'Mystery: {worst_movies_por_genero("Mystery")}%')
print(f'Family: {worst_movies_por_genero("Family")}%')
print(f'Adult: {worst_movies_por_genero("Adult")}%')
print(f'Sport: {worst_movies_por_genero("Sport")}%')
print(f'Sci-Fi: {worst_movies_por_genero("Sci-Fi")}%')

Action: 17.86%
Drama: 30.66%
Crime: 8.43%
Comedy: 30.29%
Romance: 8.42%
Mystery: 4.82%
Family: 3.2%
Adult: 2.01%
Sport: 0.65%
Sci-Fi: 7.14%


pd.DataFrame(movies['averageRating']).plot(kind='density')
plt.ylabel('Rating') 
plt.title(f"Densidad de rating para todas las películas")
plt.show()


# (promedio, desviación estándar)
(movies['averageRating'].mean(), movies['averageRating'].std())

(6.001914373707608, 1.275618572859004)


boxplot_rating_por_genero("Action")
boxplot_rating_genero_principal("Action")


boxplot_rating_por_genero("Drama")
boxplot_rating_genero_principal("Drama")


boxplot_rating_por_genero("Comedy")
boxplot_rating_genero_principal("Comedy")


dispersion_rating_duracion("Action")
dispersion_rating_duracion("Drama")
dispersion_rating_duracion("Comedy")


dispersion_rating_votos("Action")
dispersion_rating_votos("Drama")
dispersion_rating_votos("Comedy")


df = pd.DataFrame()
df["length"] = movies["primaryTitle"].apply(len)
df["startYear"] = movies["startYear"]
df["runtimeMinutes"] = movies["runtimeMinutes"]
df["averageRating"] = movies["averageRating"]
df["numVotes"] = movies["numVotes"]
corr_df = df.corr()
plt.title("Correlaciones para la tabla 'movies'")
sns.heatmap(corr_df, annot=True)
plt.show()


def etiquetarRating(rating):
    # Películas horribles
    if rating <= 5.3:
        return 0
    # Películas malas
    elif rating <= 6.1:
        return 1
    # Películas promedio
    elif rating <= 6.9:
        return 2
    # Películas excelentes
    else:
        return 3
    
def etiquetarDuracion(minutos):
    # Películas muy cortas
    if minutos <= 83:
        return 0
    # Películas cortas
    elif minutos <= 92:
        return 1
    # Películas promedio
    elif minutos <= 104:
        return 2
    # Películas largas
    else:
        return 3
    
def etiquetarAnio(anio):
    # Películas antiguas
    if anio <= 1973:
        return 0
    # Películas clásicas
    elif anio <= 2000:
        return 1
    # Películas actuales
    elif anio <= 2013:
        return 2
    # Películas nuevas
    else:
        return 3
    
def etiquetarPopularidad(numVotos):
    # Películas nada conocidas
    if numVotos <= 30:
        return 0
    # Películas más o menos conocidas
    elif numVotos <= 120:
        return 1
    # Películas conocidas
    elif numVotos <= 700:
        return 2
    # Películas famosas
    else:
        return 3
    
def etiquetarLargoTitulo(titulo):
    # Películas de nombre corto
    if titulo <= 10:
        return 0
    # Películas de nombre normal
    elif titulo <= 14:
        return 1
    # Películas de nombre largo
    elif titulo <= 20:
        return 2
    # Películas de nombre muy largo
    else:
        return 3

def run_classifier(clf, X, y, num_tests=10, test_size=.33):
    metrics = {"precision": [], "recall": [], "f1-score": []}
    for _ in range(num_tests):
        print(f"{_}/{num_tests}", end="\r")
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size)
        clf.fit(X_train, y_train.values.ravel())
        predictions = clf.predict(X_test)
        metrics["y_pred"] = predictions
        metrics["precision"].append(precision_score(y_test, predictions, average='macro', zero_division=0))
        metrics["recall"].append(recall_score(y_test, predictions, average='macro'))
        metrics["f1-score"].append(f1_score(y_test, predictions, average='macro'))
    return metrics

def sim_matrix(features, labels):
    useful_labels = labels >= 0

    indices = np.argsort(labels[useful_labels])
    sorted_features = features[useful_labels][indices]

    d = euclidean_distances(sorted_features, sorted_features)
    return d

def plot(data, model):
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(20,5))

    fig.suptitle(f"{model.__class__.__name__}")

    ax1.scatter(data[:,0], data[:,1], c=model.labels_)

    dist = sim_matrix(data, model.labels_)
    im = ax2.imshow(dist, cmap="jet")
    fig.colorbar(im, ax=ax2)


# Etiqueta rating
movies["etiquetaRating"] = movies["averageRating"].apply(etiquetarRating)

etiquetas = movies[["tconst", "directors", "genres", "numVotes", "primaryTitle", "runtimeMinutes"]]

# Etiqueta directores
etiquetas["directors"] = etiquetas["directors"].str.split(",")
etiquetas = etiquetas.explode("directors")
etiquetaDirector = preprocessing.LabelEncoder()
etiquetaDirector.fit(etiquetas.directors)
etiquetas["etiquetaDirector"] = etiquetaDirector.transform(etiquetas.directors)

# Etiqueta géneros
etiquetas["genres"] = etiquetas["genres"].str.split(",")
etiquetas = etiquetas.explode("genres")
etiquetaGenero = preprocessing.LabelEncoder()
etiquetaGenero.fit(etiquetas.genres)
etiquetas["etiquetaGenero"] = etiquetaGenero.transform(etiquetas.genres)

etiquetas = pd.merge(etiquetas, movies[['tconst', 'etiquetaRating']], on='tconst')


etiquetas["etiquetaPopularidad"] = etiquetas["numVotes"].apply(etiquetarPopularidad)
etiquetas["largoTitulo"] = etiquetas["primaryTitle"].apply(lambda x: etiquetarLargoTitulo(len(x)))
etiquetas["etiquetaDuracion"] = etiquetas["runtimeMinutes"].apply(etiquetarDuracion)


etiquetas["etiquetaPopularidad"].value_counts()

0    107700
2    106410
1    104221
3    104153
Name: etiquetaPopularidad, dtype: int64


etiquetas["largoTitulo"].value_counts()

2    121903
0    104048
3    101196
1     95337
Name: largoTitulo, dtype: int64


etiquetas["etiquetaDuracion"].value_counts()

3    109678
0    105917
1    103866
2    103023
Name: etiquetaDuracion, dtype: int64


X = etiquetas[['etiquetaDirector', 'etiquetaGenero', "etiquetaDuracion", "largoTitulo", "numVotes"]]
y = etiquetas[['etiquetaRating']]

c0 = ("Base Dummy", DummyClassifier(strategy="stratified"))
c1 = ("Decision Tree", DecisionTreeClassifier(max_depth=5))
c2 = ("K-Nearest Neighbors", KNeighborsClassifier(n_neighbors=10))
c3 = ("Gaussian Naive Bayes", GaussianNB())

classifiers = [c0, c1, c2, c3]
for name, clf in classifiers:
    metrics = run_classifier(clf, X, y)
    print(f"{name}")
    print("--------------------------")
    print(f"Precision promedio: {np.array(metrics['precision']).mean()}")
    print(f"Recall promedio: {np.array(metrics['recall']).mean()}")
    print(f"F1-Score promedio: {np.array(metrics['f1-score']).mean()}")
    print("--------------------------\n\n")

Base Dummy
--------------------------
Precision promedio: 0.2497415496254519
Recall promedio: 0.24973822997939532
F1-Score promedio: 0.24973697972920728
--------------------------


Decision Tree
--------------------------
Precision promedio: 0.3058635706634917
Recall promedio: 0.351645145897277
F1-Score promedio: 0.30723483667345886
--------------------------


K-Nearest Neighbors
--------------------------
Precision promedio: 0.41114579638428583
Recall promedio: 0.40817228610803147
F1-Score promedio: 0.40760386128253545
--------------------------


Gaussian Naive Bayes
--------------------------
Precision promedio: 0.33063289364352494
Recall promedio: 0.26753650336066415
F1-Score promedio: 0.14936150184571495
--------------------------


movies_p2 = movies.copy()
movies_p2 = movies.drop(columns=['tconst','primaryTitle','directors', 'writers', 'actors'])
movies_p2["startYear"] = movies_p2["startYear"].apply(etiquetarAnio)
movies_p2["runtimeMinutes"] = movies_p2["runtimeMinutes"].apply(etiquetarDuracion)

# Etiqueta géneros
movies_p2["genres"] = movies_p2["genres"].str.split(",")
movies_p2 = movies_p2.explode("genres")
etiquetaGenero = preprocessing.LabelEncoder()
etiquetaGenero.fit(movies_p2.genres)
movies_p2["genres"] = etiquetaGenero.transform(movies_p2.genres)

# Etiqueta de rating y de votos
movies_p2["averageRating"] = movies_p2["averageRating"].apply(etiquetarRating)
movies_p2["numVotes"] = movies_p2["numVotes"].apply(etiquetarPopularidad)

size = movies_p2.shape[0]


X = movies_p2

sse = []

clusters = list(range(1, 16))
for k in clusters:
    kmeans = KMeans(n_clusters=k).fit(X)
    sse.append(kmeans.inertia_)

plt.plot(clusters, sse, marker="o")
plt.title("Método del codo de 1 a 15 clusters")
plt.grid(True)
plt.show()


new_X = PCA(n_components=2, random_state=0).fit_transform(X)
kmeans = KMeans(n_clusters=2, random_state=20).fit(movies_p2)

kmeans.fit(X)
kpredict = kmeans.predict(X)

plt.scatter(new_X[:, 0], new_X[:, 1], c=kmeans.labels_)
plt.title("Representación de clusters, en 2 dimensiones")
plt.show()


print("Dataset X K-Means 2\t", silhouette_score(X, kmeans.labels_))

Dataset X K-Means 2	 0.5886217357191276


Y = movies_p2.sample(int(size*0.01))

complete = linkage(Y, method="complete")
single = linkage(Y, method="single")
average = linkage(Y, method="average")
ward = linkage(Y, method="ward")

dendrogram(complete)
plt.title("Linkage: Complete")
plt.show()

dendrogram(single)
plt.title("Linkage: Single")
plt.show()

dendrogram(average)
plt.title("Linkage: Average")
plt.show()

dendrogram(ward)
plt.title("Linkage: Ward")
plt.show()


new_Y = PCA(n_components=2, random_state=0).fit_transform(Y)

agc_complete = AgglomerativeClustering(n_clusters=None,linkage="complete",distance_threshold=8).fit(Y)
print("Aglomerative clustering, complete linkage: clusters generated=", agc_complete.n_clusters_)
print(pd.DataFrame(agc_complete.labels_).stack().value_counts())

complete = AgglomerativeClustering(linkage="complete", n_clusters=8).fit(new_Y)
plot(new_Y, complete)
plt.show()

Aglomerative clustering, complete linkage: clusters generated= 7
0    1305
1     890
5     510
4     329
2     318
6     274
3     154
dtype: int64


agc_average = AgglomerativeClustering(n_clusters=None,linkage="average",distance_threshold=4).fit(Y)
print("Aglomerative clustering, average linkage: clusters generated=", agc_average.n_clusters_)
print(pd.DataFrame(agc_average.labels_).stack().value_counts())

average = AgglomerativeClustering(linkage="average", n_clusters=9).fit(new_Y)

plot(new_Y, average)
plt.show()

Aglomerative clustering, average linkage: clusters generated= 11
5     1328
0      890
3      374
1      327
2      254
4      185
6      132
8      114
10      96
9       77
7        3
dtype: int64


agc_ward = AgglomerativeClustering(n_clusters=None,linkage="ward",distance_threshold=50).fit(Y)
print("Aglomerative clustering, ward linkage: clusters generated=", agc_ward.n_clusters_)
print(pd.DataFrame(agc_ward.labels_).stack().value_counts())

ward = AgglomerativeClustering(linkage="ward", n_clusters=7).fit(new_Y)

plot(new_Y, ward)
plt.show()

Aglomerative clustering, ward linkage: clusters generated= 8
0    872
1    488
6    455
3    455
7    435
2    428
5    327
4    320
dtype: int64


Y = movies_p2.sample(int(size*0.1))
new_Y = PCA(n_components=2, random_state=0).fit_transform(Y)

nbrs = NearestNeighbors(n_neighbors=5).fit(new_Y)
distances, indices = nbrs.kneighbors(new_Y)

distances = np.sort(distances, axis=0)
distances = distances[:,1]
plt.axhline(y=0.01, color='r', linestyle='--') #Ajuste el valor para "y" en esta línea
plt.ylim(0,0.0000002)
plt.plot(distances)

[<matplotlib.lines.Line2D at 0x2939cb60490>]


Y = movies_p2.sample(int(size*0.1))
new_Y = PCA(n_components=2, random_state=0).fit_transform(Y)

eps = 0.001
min_samples = 10

dbscan = DBSCAN(eps=eps, min_samples=min_samples).fit(new_Y)
plt.scatter(new_Y[:, 0], new_Y[:, 1], c=dbscan.labels_)

plt.show()


_filter_label = dbscan.labels_ >= 0
print("Dataset Y DBSCAN\t", silhouette_score(new_Y[_filter_label], dbscan.labels_[_filter_label]))

Dataset Y DBSCAN	 0.9999998470064091


def etiquetarPresenciaActor(nombre, inicio_carrera=1800, fin_carrera=3000):
    def etiquetarAnioPorActor(anio):
        return 1 if inicio_carrera <= anio and anio <= fin_carrera else 0
    moviesActor = movies.copy()
    moviesActor["actors"] = moviesActor["actors"].astype(str)
    moviesActor["etiquetaActor"] = moviesActor["startYear"]
    moviesActor["etiquetaActor"] = moviesActor["etiquetaActor"].apply(etiquetarAnioPorActor)
    moviesActor = moviesActor.loc[moviesActor['etiquetaActor'] == 1]
    
    actorsDf = moviesActor[["tconst", "actors"]]
    actorsDf["etiquetaActor"] = actorsDf["actors"].apply(lambda x: nombre in x)
    actorsDf = actorsDf.drop(columns=["actors"])
    
    moviesCopy = movies.copy()
    moviesActor = pd.merge(moviesCopy, actorsDf, on='tconst', how="left")
    moviesActor['etiquetaActor'] = moviesActor["etiquetaActor"].fillna(False)
    moviesActor["largoTitulo"] = moviesActor["primaryTitle"].apply(lambda x: etiquetarLargoTitulo(len(x)))
    moviesActor["etiquetaDuracion"] = moviesActor["runtimeMinutes"].apply(etiquetarDuracion)
    moviesActor["etiquetaAnio"] = moviesActor["startYear"].apply(etiquetarAnio)
    moviesActor["etiquetaPopularidad"] = moviesActor["numVotes"].apply(etiquetarPopularidad)
    
    return moviesActor


tabla = etiquetarPresenciaActor("Brad Pitt", 1987)
X = tabla[['etiquetaActor', 'largoTitulo', "etiquetaAnio", "etiquetaDuracion", "etiquetaRating"]]
y = tabla[['etiquetaPopularidad']]

c0 = ("Base Dummy", DummyClassifier(strategy="stratified"))
c1 = ("Decision Tree", DecisionTreeClassifier(max_depth=5))
c2 = ("K-Nearest Neighbors", KNeighborsClassifier(n_neighbors=10))
c3 = ("Gaussian Naive Bayes", GaussianNB())

classifiers = [c0, c1, c2, c3]
for name, clf in classifiers:
    metrics = run_classifier(clf, X, y)
    print(f"{name}")
    print("--------------------------")
    print(f"Precision promedio: {np.array(metrics['precision']).mean()}")
    print(f"Recall promedio: {np.array(metrics['recall']).mean()}")
    print(f"F1-Score promedio: {np.array(metrics['f1-score']).mean()}")
    print("--------------------------\n\n")

Base Dummy
--------------------------
Precision promedio: 0.2499382938572004
Recall promedio: 0.2499420547042679
F1-Score promedio: 0.24992896131545192
--------------------------


Decision Tree
--------------------------
Precision promedio: 0.3595157716316776
Recall promedio: 0.36752153327103304
F1-Score promedio: 0.351533932493619
--------------------------


K-Nearest Neighbors
--------------------------
Precision promedio: 0.3253361100656407
Recall promedio: 0.32243938896406255
F1-Score promedio: 0.31893289295092125
--------------------------


Gaussian Naive Bayes
--------------------------
Precision promedio: 0.42018996357229615
Recall promedio: 0.29509764811042855
F1-Score promedio: 0.21064428689119316
--------------------------

	tconst	primaryTitle	isAdult	startYear	runtimeMinutes	genres	averageRating	numVotes	directors	writers
0	tt0000574	The Story of the Kelly Gang	0	1906.0	70.0	Action,Adventure,Biography	6.1	692	nm0846879	nm0846879
1	tt0000591	The Prodigal Son	0	1907.0	90.0	Drama	5.2	16	nm0141150	nm0141150
2	tt0000679	The Fairylogue and Radio-Plays	0	1908.0	120.0	Adventure,Fantasy	5.4	65	nm0877783,nm0091767	nm0000875,nm0877783
3	tt0001184	Don Juan de Serrallonga	0	1910.0	58.0	Adventure,Drama	3.4	19	nm0063413,nm0550220	nm0049370
4	tt0001258	The White Slave Trade	0	1910.0	45.0	Drama	5.7	132	nm0088881	\N
...	...	...	...	...	...	...	...	...	...	...
235204	tt9916170	The Rehearsal	0	2019.0	51.0	Drama	7.2	5	nm5412267	nm5412267,nm6743460,nm3245789
235205	tt9916190	Safeguard	0	2020.0	90.0	Action,Adventure,Thriller	3.6	222	nm7308376	nm7308376
235206	tt9916270	Il talento del calabrone	0	2020.0	84.0	Thriller	5.8	1163	nm1480867	nm1480867,nm10538402
235207	tt9916362	Coven	0	2020.0	92.0	Adventure,Drama,History	6.3	3829	nm1893148	nm1893148,nm3471432
235208	tt9916538	Kuambil Lagi Hatiku	0	2019.0	123.0	Drama	8.3	6	nm4457074	nm4843252,nm4900525,nm2679404

	tconst	directors
0	tt0000574	Charles Tait
1	tt0000591	Michel Carré
2	tt0000679	Otis Turner,Francis Boggs
3	tt0001184	Ricardo de Baños,Alberto Marro
4	tt0001258	August Blom
...	...	...
235204	tt9916170	Tamar Guimaraes
235205	tt9916190	Fraser Precious
235206	tt9916270	Giacomo Cimini
235207	tt9916362	Pablo Agüero
235208	tt9916538	Azhar Kinoi Lubis

	tconst	primaryTitle	isAdult	startYear	runtimeMinutes	genres	averageRating	numVotes	directors
0	tt0000574	The Story of the Kelly Gang	0	1906.0	70.0	Action,Adventure,Biography	6.1	692	Charles Tait
1	tt0000591	The Prodigal Son	0	1907.0	90.0	Drama	5.2	16	Michel Carré
2	tt0000679	The Fairylogue and Radio-Plays	0	1908.0	120.0	Adventure,Fantasy	5.4	65	Otis Turner,Francis Boggs
3	tt0001184	Don Juan de Serrallonga	0	1910.0	58.0	Adventure,Drama	3.4	19	Ricardo de Baños,Alberto Marro
4	tt0001258	The White Slave Trade	0	1910.0	45.0	Drama	5.7	132	August Blom
...	...	...	...	...	...	...	...	...	...
235204	tt9916170	The Rehearsal	0	2019.0	51.0	Drama	7.2	5	Tamar Guimaraes
235205	tt9916190	Safeguard	0	2020.0	90.0	Action,Adventure,Thriller	3.6	222	Fraser Precious
235206	tt9916270	Il talento del calabrone	0	2020.0	84.0	Thriller	5.8	1163	Giacomo Cimini
235207	tt9916362	Coven	0	2020.0	92.0	Adventure,Drama,History	6.3	3829	Pablo Agüero
235208	tt9916538	Kuambil Lagi Hatiku	0	2019.0	123.0	Drama	8.3	6	Azhar Kinoi Lubis

	tconst	writers
0	tt0000574	Charles Tait
1	tt0000591	Michel Carré
2	tt0000679	L. Frank Baum,Otis Turner
3	tt0001184	Víctor Balaguer
4	tt0001258
...	...	...
235204	tt9916170	Tamar Guimaraes,Lillah Halla,Melissa de Raaf
235205	tt9916190	Fraser Precious
235206	tt9916270	Giacomo Cimini,Lorenzo Collalti
235207	tt9916362	Pablo Agüero,Katell Guillou
235208	tt9916538	Arief Ash Siddiq,Rino Sarjono,Salman Aristo

	tconst	primaryTitle	isAdult	startYear	runtimeMinutes	genres	averageRating	numVotes	directors	writers
0	tt0000574	The Story of the Kelly Gang	0	1906.0	70.0	Action,Adventure,Biography	6.1	692	Charles Tait	Charles Tait
1	tt0000591	The Prodigal Son	0	1907.0	90.0	Drama	5.2	16	Michel Carré	Michel Carré
2	tt0000679	The Fairylogue and Radio-Plays	0	1908.0	120.0	Adventure,Fantasy	5.4	65	Otis Turner,Francis Boggs	L. Frank Baum,Otis Turner
3	tt0001184	Don Juan de Serrallonga	0	1910.0	58.0	Adventure,Drama	3.4	19	Ricardo de Baños,Alberto Marro	Víctor Balaguer
4	tt0001258	The White Slave Trade	0	1910.0	45.0	Drama	5.7	132	August Blom
...	...	...	...	...	...	...	...	...	...	...
235204	tt9916170	The Rehearsal	0	2019.0	51.0	Drama	7.2	5	Tamar Guimaraes	Tamar Guimaraes,Lillah Halla,Melissa de Raaf
235205	tt9916190	Safeguard	0	2020.0	90.0	Action,Adventure,Thriller	3.6	222	Fraser Precious	Fraser Precious
235206	tt9916270	Il talento del calabrone	0	2020.0	84.0	Thriller	5.8	1163	Giacomo Cimini	Giacomo Cimini,Lorenzo Collalti
235207	tt9916362	Coven	0	2020.0	92.0	Adventure,Drama,History	6.3	3829	Pablo Agüero	Pablo Agüero,Katell Guillou
235208	tt9916538	Kuambil Lagi Hatiku	0	2019.0	123.0	Drama	8.3	6	Azhar Kinoi Lubis	Arief Ash Siddiq,Rino Sarjono,Salman Aristo

Hito 3: Caracterización y análisis de películas de IMDB¶

CC5206 Semestre Primavera 2021¶

Introducción¶

Mejoras al Hito 1¶

Dataset¶

Columnas nuevas¶

Reformulación de preguntas y problemas¶

Propuesta experimental¶

Funciones útiles¶

Pregunta 1¶

Pre - procesamiento¶

Resultados¶

Pregunta 2¶

Pre - procesamiento¶

Resultados¶

Pregunta 3¶

Pre - procesamiento¶

Resultados¶

Conclusiones generales¶

Planificación futura¶

Contribución de cada miembro¶

Alan Acevedo:¶

Camila Labarca:¶

Franco Miranda:¶

Julia Paredes:¶

	tconst	actors
0	tt0000574	Elizabeth Tait,John Tait,Norman Campbell,Bella...
1	tt0000591	Georges Wague,Henri Gouget,Christiane Mandelys...
2	tt0000679	L. Frank Baum,Frank Burns,George E. Wilson,Wal...
3	tt0001184	Dolores Puchol,Cecilio Rodríguez de la Vega
4	tt0001258	Aage Lorentzen,Ellen Diedrich,Victor Fabian,Ju...
...	...	...
207478	tt9916170	Julia Ianina,Pablo Lafuente,Kelner Macêdo,Germ...
207479	tt9916190	Patrick Gallagher,Akie Kotabe,Takayuki Suzuki,...
207480	tt9916270	Sergio Castellitto,Lorenzo Richelmy,Anna Fogli...
207481	tt9916362	Amaia Aberasturi,Alex Brendemühl,Daniel Fanego...
207482	tt9916538	Lala Karmela,Cut Mini Theo,Sahil Shah,Ria Irawan

	startYear	runtimeMinutes	averageRating	numVotes
count	207483.000000	207483.000000	207483.000000	2.074830e+05
mean	1991.278119	95.541558	6.001914	4.363440e+03
std	26.041480	103.329849	1.275619	3.730290e+04
min	1906.000000	1.000000	1.000000	5.000000e+00
25%	1973.000000	83.000000	5.300000	2.500000e+01
50%	2000.000000	92.000000	6.100000	8.800000e+01
75%	2013.000000	104.000000	6.900000	4.410000e+02
max	2021.000000	43200.000000	10.000000	2.455855e+06

	tconst	primaryTitle	startYear	runtimeMinutes	genres	averageRating	numVotes	directors	writers	actors
139637	tt14923112	Methagu	2021.0	100.0	Biography,History	9.6	8416	Kittu	Muthu Cheziyan,Kittu,Thirukumaran	Lizzie Antony,Eshwar Basha,Chandrasekar,Kutti ...
86360	tt0252487	The Chaos Class	1975.0	87.0	Comedy,Drama	9.3	38992	Ertem Egilmez	Umur Bugay,Rifat Ilgaz	Kemal Sunal,Münir Özkul,Halit Akçatepe,Tarik Akan
56429	tt0111161	The Shawshank Redemption	1994.0	142.0	Drama	9.3	2455855	Frank Darabont	Stephen King,Frank Darabont	Tim Robbins,Morgan Freeman,Bob Gunton,William ...
33891	tt0068646	The Godfather	1972.0	175.0	Crime,Drama	9.2	1699389	Francis Ford Coppola	Mario Puzo,Francis Ford Coppola	Marlon Brando,Al Pacino,James Caan,Diane Keaton
127660	tt11737772	Chal Mera Putt 2	2020.0	124.0	Drama	9.2	2617	Janjot Singh	Rakesh Dhawan	Amrinder Gill,Simi Chahal,Nasir Chinyoti,Garry...
103705	tt0383177	Aguner Poroshmoni	1994.0	123.0	Drama,War	9.1	2991	Humayun Ahmed	Humayun Ahmed	Bipasha Hayat,Asaduzzaman Noor,Abul Hayat,Doll...
184903	tt5354160	Mirror Game	2016.0	147.0	Crime,Mystery,Thriller	9.1	24835	Amitabh Reza Chowdhury	Syed Gaosul Alam Shaon,Anam Biswas,Adnan Adib ...	Chanchal Chowdhury,Masuma Rahman Nabila,Partha...
161918	tt2592910	CM101MMXI Fundamentals	2013.0	139.0	Comedy,Documentary	9.1	44645	Murat Dündar	Cem Yilmaz	Bilal Kaya,Caner Tüfek
120243	tt10189514	Soorarai Pottru	2020.0	153.0	Drama	9.1	89377	Sudha Kongara	Sudha Kongara,Shalini Ushadevi,Vijay Kumar,Aal...	Suriya,Paresh Rawal,Aparna Balamurali,Prakash ...
122101	tt10534500	#Home	2021.0	158.0	Drama	9.1	7949	Rojin Thomas	Rojin Thomas	Indrans,Sreenath Bhasi,Manju Pillai,Naslen

	tconst	primaryTitle	startYear	runtimeMinutes	genres	averageRating	numVotes	directors	writers	actors
141178	tt1538949	Play in the Gray	2009.0	85.0	Biography,Comedy,Documentary	1.0	132	Kaitlin Meelia	Ian McFarland	Katie Allen,Julee Antonellis,Kate Bornstein,Le...
142980	tt1611056	Hito no sabaku	2010.0	121.0	Drama	1.0	475	Hiroshi Gokan,Shinsuke Kurimoto,Paul Young,Dai...	Kôtarô Sawaki,Akiko Amy Kanda,Takashi Hattori,...	Masashi Arifuku,Yôjin Hino,Ben Hiura,Renji Ish...
150620	tt1945118	In a Lonely Planet	2011.0	94.0	Romance	1.0	459	Takefumi Tsutsui	Daisuke Miyazaki	Yui Asano,Gô Ayano,Mickey Curtis,Takaaki Ichiyama
120737	tt1028548	Konjaku monogatari: The new edition	2007.0	88.0	Comedy,Drama	1.0	500	Mikio Ohkado,Shô Tsukikawa,Yûko Watanabe,Naoki...	Naoki Katô,Michiko Ohishi,Sachiko Tanaka,Kiyoh...	Akino,Azusa,Shusaku Fujiwara,Yinling of Joytoy
153666	tt2100624	Good Morning to the World!!	2010.0	81.0	Comedy	1.0	458	Satoru Hirohara	Satoru Hirohara	Miho Arai,Mitsunori Izumi,Shotaro Kaneyama,Kôi...
133641	tt13232730	Hearts Are Trump	2020.0	112.0	Drama	1.0	21	Donald James Parker	Donald James Parker	Danielle Superior,William Row,John Goad,Juli T...
199889	tt7923374	Badang	2018.0	105.0	Action,Comedy,Fantasy	1.0	742	A. Razak Mohaideen	A. Razak Mohaideen,Azlan Syah,Azhari Mohd Zain	Aliff Syukr,Fasha Sandha,Azhari Mohd Zain,Fauz...
158948	tt2370036	On BorrowedTime	2012.0	60.0	Documentary	1.0	12	Emilio Roso	Emilio Roso	Emilio Roso,John Jacobs
134262	tt13351868	Bootleg Death Tape III	2020.0	45.0	Horror	1.0	8	Sean Murray,Jamie Robert MacDougall,Ian Boyd,T...	Ian Boyd,Avery Crumley,Valeria Henry,Jamie Rob...	Tony Newton,Avery Crumley,Jamie Robert MacDoug...
202271	tt8476266	Teambuilding	2018.0	83.0	Comedy	1.0	13	Ján Novák	Daniela Choderová,Ján Novák	Jan Adámek,Alice Bendová,Hynek Chmelar,Daniela...