%%time
# installing the sentence-transformers and gensim libraries for word embeddings
!pip install -U sentence-transformers gensim transformers tqdm -q

%%time
# To manipulate and analyze data
import pandas as pd
import numpy as np

# To visualize data
import matplotlib.pyplot as plt
import seaborn as sns

# To used time-related functions
import time

# To parse JSON data
import json

# To build, tune, and evaluate ML models
# from sklearn.ensemble import DecisionTreeClassifier # Incorrecct import
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier # Update import replacing commented sklearn.ensemble import

from sklearn.model_selection import GridSearchCV
from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, precision_score, recall_score

# To load/create word embeddings
from gensim.models import Word2Vec
from gensim.models import KeyedVectors
from gensim.scripts.glove2word2vec import glove2word2vec

# To work with transformer models
import torch
from sentence_transformers import SentenceTransformer

# To implement progress bar related functionalities
from tqdm import tqdm
tqdm.pandas()

# To ignore unnecessary warnings
import warnings
warnings.filterwarnings('ignore')

# Check library versions
print("pandas:   ", pd.__version__)
print("numpy:    ", np.__version__)
print("seaborn:  ", sns.__version__)
print("torch:    ", torch.__version__)

from google.colab import drive # Mount drive
drive.mount('/content/drive')  # Dataset is in Google Drive

stock_news = pd.read_csv("/content/stock_news.csv") # Read the CSV file.

stock = stock_news.copy() #Create a copy of the dataset

stock.sample(n=10, random_state=42) # Randomly select 10 rows and set a random seed for reproducibility (i.e., to always get the same random rows each time you run the code).

stock.shape # Check the shape of the data

stock.dtypes # Check the data types

stock['Date'] = pd.to_datetime(stock['Date'])  # Convert the 'Date' column in the 'stock' DataFrame to datetime format.

stock.dtypes # Notice 'Date' column converted from object to datetime format.

pd.set_option('display.float_format', '{:.2f}'.format) # Set display options to show 2 decimal places
stock.describe() # Check the statistical summary

stock.duplicated().sum() #Complete the code to check the duplicate values

stock.isnull().sum() # Complete the code to check for missing values in the data

sns.set_palette("Purples")                                   # Set color to violet for the bars.

plt.figure(figsize=(2, 4))                                   # Adjust width and height.

ax = sns.countplot(data=stock, x="Label", stat="percent")    # Create the count plot

# Add percentage values at the top of each bar
total = len(stock)                                           # Total number of items in the dataset
for p in ax.patches:
    height = p.get_height()
    ax.annotate(f'{height:.2f}%',                            # Display percentage with 2 decimals
                (p.get_x() + p.get_width() / 2., height),    # Position at the top of the bar
                ha='center', va='center',                    # Alignment
                fontsize=8, color='blue', fontweight='bold', # Styling
                xytext=(0, 5), textcoords='offset points')   # Adjust text position

plt.show()                                                   # Show the plot

# Density Plot of Price (Open,High,Low,Close)
for i in ["Open","High","Low","Close"]:
    sns.kdeplot(stock[i], label=i, shade=True)
plt.xlabel("Price")
plt.ylabel("Density")
plt.title("Density Plot of Stock Prices")
plt.legend()
plt.show()

sns.displot(data=stock[['Open','High','Low','Close']], kind="kde", palette="tab10"); # Plot a density plot of ["Open","High","Low","Close"] all in a single plot

#Plot a histogram of Volume
plt.figure(figsize=(8, 2))
sns.histplot(stock['Volume'], bins=40, kde=True)
plt.xlabel('Volume')
plt.ylabel('Frequency')
plt.title('Distribution of Volume')
plt.show()

stock['news_len'] = stock['News'].apply(lambda x: len(x.split(' '))) # Calculating the total number of words present in the news content column.
stock['news_len'].describe()                                         # Print the statistical summary for the news content length after splitting into words.

stock.dtypes # Observe which columns have numerical data

numeric_columns = stock.select_dtypes(include=['number']) # Filter only numeric columns

# Plot the heatmap for numeric columns only
sns.heatmap(
    numeric_columns.corr(), annot=True, vmin=-1, vmax=1, fmt=".2f", cmap="Spectral"
)
plt.show()

plt.figure(figsize=(8, 4))

for i, variable in enumerate(['Open', 'High', 'Low', 'Close']):
    plt.subplot(2, 2, i + 1)
    sns.boxplot(data=stock, x="Label", y=variable) # Label = Sentiment (1) - Positive, (0) - Neutral, (-1) - Negative
    plt.tight_layout(pad=2)

plt.show()

sns.boxplot(
    data=stock, x="Label", y="Volume"  # # Label = Sentiment (1) - Positive, (0) - Neutral, (-1) - Negative
);

stock_daily = stock.groupby('Date').agg(
    {
        'Open': 'mean',
        'High': 'mean',
        'Low': 'mean',
        'Close': 'mean',
        'Volume': 'mean',
    }
).reset_index()                             # Group the 'stocks' DataFrame by the 'Date' column

stock_daily.set_index('Date', inplace=True) # Index
stock_daily.sample(n=10, random_state=42)   # Random selection of rows. But the same ones used whenever randon is chosen.

plt.figure(figsize=(12,3))
sns.lineplot(stock_daily.drop("Volume", axis=1)); # Lineplot of variables except Volume

fig, ax1 = plt.subplots(figsize=(15,5))      # Create a figure and axis
sns.lineplot(data=stock_daily.reset_index(), x='Date', y='Close', ax=ax1, color='blue', marker='o', label='Close Price') # Lineplot on primary y-axis
ax2 = ax1.twinx()                            # Create a secondary y-axis
sns.lineplot(data=stock_daily.reset_index(), x='Date', y='Volume', ax=ax2, color='gray', marker='o', label='Volume') # Lineplot on secondary y-axis
ax1.legend(bbox_to_anchor=(1,1));            # Legend set to the Volume data

pd.set_option('display.float_format', '{:.2f}'.format) # Set display options to show 2 decimal places
stock.describe() # Check the statistical summary

stock["Date"].fillna("Unknown", inplace=True) # Replace NaN with a Default Value (e.g., “Unknown” or a placeholder)
stock["Date"].describe()

X_train = stock[(stock['Date'] < '2019-04-01')].reset_index()    # Select all rows where the 'Date' is before '2019-04-01'
X_val = stock[(stock['Date'] >= '2019-04-01') & (stock['Date'] < '2019-04-16')].reset_index()    # Select all rows where the 'Date' is from '2019-04-01 to '2019-04-16' (excluded)
X_test = stock[stock['Date'] >= '2019-04-16'].reset_index()      # Select all rows where the 'Date' is from '2019-04-16' till the end.

# 'Label' column is the target variable (lower cse variables)
y_train = X_train["Label"].copy()
y_val = X_val["Label"].copy()
y_test = X_test["Label"].copy()

# Print the shape of X_train,X_val,X_test,y_train,y_val and y_test
print("\nTrain data shape",X_train.shape)
print("Validation data shape",X_val.shape)
print("Test data shape ",X_test.shape)
line= '_' * 25
print(line)
print("\nTrain Label shape",y_train.shape)
print("Validation Label shape",y_val.shape)
print("Test Label shape ",y_test.shape)

words_list = [item.split(" ") for item in stock['News'].values] # Creating a list of all words in our data

# Creating an instance of Word2Vec
vec_size = 300 # Determines the number of features used to represent each word in the vector space. A higher vec_size can increase computational complexity as it captures more nuances.
model_W2V = Word2Vec(words_list, vector_size = vec_size, min_count = 1, window=5, workers = 6) # Model will learn these embeddings by analyzing word co-occurrences within a context window of 5 words.

print("Length of the vocabulary is", len(list(model_W2V.wv.key_to_index))) # Size of the vocabulary or number of unique words that the Word2Vec model has learned representations for.

word = "stock"     # Selected word used frequently.
model_W2V.wv[word] # Observe the word embedding of a selected word

word = "economy"   # Second selected word
model_W2V.wv[word] # Observe the word embedding of the second selected word

words = list(model_W2V.wv.key_to_index.keys()) # Retrieve the words present in the --Word2Vec-- model's vocabulary
wvs = model_W2V.wv[words].tolist()             # Retrieve word vectors for all the words present in the model's vocabulary
word_vector_dict = dict(zip(words, wvs))       # Create a dictionary of words and their corresponding vectors

def average_vectorizer_Word2Vec(doc):
    # Initializing a feature vector for the sentence
    feature_vector = np.zeros((vec_size,), dtype="float64")

    # Create a list of words in the sentence that are present in the model vocabulary
    words_in_vocab = [word for word in doc.split() if word in words]

    # Add the vector representations of the words
    for word in words_in_vocab:
        feature_vector += np.array(word_vector_dict[word])

    # Divide by the number of words to get the average vector
    if len(words_in_vocab) != 0:
        feature_vector /= len(words_in_vocab)

    return feature_vector

# Create a dataframe of the vectorized documents
start = time.time()

X_train_wv = pd.DataFrame(X_train["News"].apply(average_vectorizer_Word2Vec).tolist(), columns=['Feature '+str(i) for i in range(vec_size)])
X_val_wv = pd.DataFrame(X_val["News"].apply(average_vectorizer_Word2Vec).tolist(), columns=['Feature '+str(i) for i in range(vec_size)])
X_test_wv = pd.DataFrame(X_test["News"].apply(average_vectorizer_Word2Vec).tolist(), columns=['Feature '+str(i) for i in range(vec_size)])

end = time.time()
print('Time taken ', (end-start))

print(X_train_wv.shape,'train split\n', X_val_wv.shape,'validation split\n', X_test_wv.shape,'test split\n') # Train-Validatio-Test Splits

# Download the GloVe model (Stanford's) if it doesn't exist
!wget http://nlp.stanford.edu/data/glove.6B.zip
!unzip glove.6B.zip

# Convert GloVe to word2vec format
glove_input_file = 'glove.6B.100d.txt'
word2vec_output_file = 'glove.6B.100d.txt.word2vec'
glove2word2vec(glove_input_file, word2vec_output_file)

# Load the converted model
filename = 'glove.6B.100d.txt.word2vec'
glove_model = KeyedVectors.load_word2vec_format(filename, binary=False)

print("Length of the vocabulary is", len(glove_model.index_to_key)) # Check the size of the vocabulary

word = "stock"    # Select the word embedding for first word. A very frequently used word.
glove_model[word] # View the word embedding of selected word

word = "economy"  # Select the word embedding for a second word.
glove_model[word] # View the word embedding of selected word

glove_words = glove_model.index_to_key                                                 # Retrieve the words present in the GloVe model's vocabulary
glove_word_vector_dict = dict(zip(glove_model.index_to_key,list(glove_model.vectors))) # Create a dictionary of words and their corresponding vectors

# Each word can be represented by a 100-dimensional vector (100 features).
vec_size=100 # Specifies the number of dimensions for the embedding space.

def average_vectorizer_GloVe(doc):
    # Initializing a feature vector for the sentence
    feature_vector = np.zeros((vec_size,), dtype="float64")

    # Creating a list of words in the sentence that are present in the model vocabulary
    words_in_vocab = [word for word in doc.split() if word in glove_words]

    # adding the vector representations of the words
    for word in words_in_vocab:
        feature_vector += np.array(glove_word_vector_dict[word])

    # Dividing by the number of words to get the average vector
    if len(words_in_vocab) != 0:
        feature_vector /= len(words_in_vocab)

    return feature_vector

# Create a dataframe of the vectorized documents
start = time.time()

X_train_gl = pd.DataFrame(X_train["News"].apply(average_vectorizer_GloVe).tolist(), columns=['Feature '+str(i) for i in range(vec_size)]) # Apply GloVe on 'News' column for Training set.
X_val_gl = pd.DataFrame(X_val["News"].apply(average_vectorizer_GloVe).tolist(), columns=['Feature '+str(i) for i in range(vec_size)])     # Apply GloVe on 'News' column For Validation set.
X_test_gl = pd.DataFrame(X_test["News"].apply(average_vectorizer_GloVe).tolist(), columns=['Feature '+str(i) for i in range(vec_size)])   # Apply GloVe on 'News' column for Testing set.

end = time.time()
print('Time taken ', (end-start))

print(f'Time taken to create a dataframe of the vectorized documents using GloVe \033[1m{end - start:.6f} seconds.') # Rounded to 6 significant digits.

print('For GloVe:\n', X_train_gl.shape,'train split\n', X_val_gl.shape,'validation split\n', X_test_gl.shape,'test split\n') # Train-Validatio-Test Splits

model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2') # Defining the model for text classification, semantic search and sentiment analysis.

device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # Setting the device to GPU if available, else CPU

# Encoding the dataset splits with the Transformer Model
start = time.time()

X_train_st = model.encode(X_train["News"].values, show_progress_bar=True, device=device) # Apply Sentence Transformer on 'News' column for the Training set.
X_val_st = model.encode(X_val["News"].values, show_progress_bar=True, device=device)     # Apply Sentence Transformer on 'News' column for the Validation set.
X_test_st = model.encode(X_test["News"].values, show_progress_bar=True, device=device)   # Apply Sentence Transformer on 'News' column for the Test set.

end = time.time()
print(f'Time taken using a Transformer \033[1m{end - start:.6f} seconds.') # Rounded to 6 significant digits.

print('For our Transformer model:\n', X_train_st.shape,'train split\n', X_val_st.shape,'validation split\n', X_test_st.shape,'test split\n') # Train-Validatio-Test Splits

def plot_confusion_matrix(model, predictors, target):
    """
    Plot a confusion matrix to visualize the performance of a classification model.

    Parameters:
    actual (array-like): The true labels.
    predicted (array-like): The predicted labels from the model.

    Returns:
    None: Displays the confusion matrix plot.
    """
    pred = model.predict(predictors)  # Make predictions using the classifier.

    cm = confusion_matrix(target, pred)  # Compute the confusion matrix.

    plt.figure(figsize=(5, 4))  # Create a new figure with a specified size.
    label_list = [0, 1,-1]  # Define the labels for the confusion matrix.
    sns.heatmap(cm, annot=True, fmt='.0f', cmap='Blues', xticklabels=label_list, yticklabels=label_list)
    # Plot the confusion matrix using a heatmap with annotations.

    plt.ylabel('Actual')  # Label for the y-axis.
    plt.xlabel('Predicted')  # Label for the x-axis.
    plt.title('Confusion Matrix')  # Title of the plot.
    plt.show()  # Display the plot.

def model_performance_classification_sklearn(model, predictors, target):
    """
    Compute various performance metrics for a classification model using sklearn.

    Parameters:
    model (sklearn classifier): The classification model to evaluate.
    predictors (array-like): The independent variables used for predictions.
    target (array-like): The true labels for the dependent variable.

    Returns:
    pandas.DataFrame: A DataFrame containing the computed metrics (Accuracy, Recall, Precision, F1-score).
    """
    pred = model.predict(predictors)  # Make predictions using the classifier.

    acc = accuracy_score(target, pred)  # Compute Accuracy.
    recall = recall_score(target, pred,average='weighted')  # Compute Recall.
    precision = precision_score(target, pred,average='weighted')  # Compute Precision.
    f1 = f1_score(target, pred,average='weighted')  # Compute F1-score.

    # Create a DataFrame to store the computed metrics.
    df_perf = pd.DataFrame(
        {
            "Accuracy": [acc],
            "Recall": [recall],
            "Precision": [precision],
            "F1": [f1],
        }
    )

    return df_perf  # Return the DataFrame with the metrics.

# Building the model

#Uncomment only one of the snippets related to fitting the model to the data

#base_wv = GradientBoostingClassifier(random_state = 42)
#base_wv = RandomForestClassifier(random_state=42)
base_wv = DecisionTreeClassifier(random_state=42)

# Fitting on train data
base_wv.fit(X_train_wv, y_train)

plot_confusion_matrix(base_wv,X_train_wv,y_train) # Training

plot_confusion_matrix(base_wv,X_val_wv,y_val) # Validation

# Calculating different metrics on training data
base_train_wv = model_performance_classification_sklearn(base_wv,X_train_wv,y_train)
print("Training performance:\n", base_train_wv)

# Calculating different metrics on validation data
base_val_wv = model_performance_classification_sklearn(base_wv,X_val_wv,y_val)
print("Validation performance:\n",base_val_wv)

#Building the model

#Uncomment only one of the snippets related to fitting the model to the data

#base_wv = GradientBoostingClassifier(random_state = 42)
#base_wv = RandomForestClassifier(random_state=42)
base_gl = DecisionTreeClassifier(random_state=42)

# Fitting on train data
base_gl.fit(X_train_gl, y_train) #Complete the code to fit the chosen model on the train data

plot_confusion_matrix(base_gl,X_train_gl,y_train) # Confusion matrix for the train data on GloVe

plot_confusion_matrix(base_gl,X_val_gl,y_val) # Confusion matrix for the validation data on GloVe

#Calculating different metrics on training data
base_train_gl=model_performance_classification_sklearn(base_gl,X_train_gl,y_train) # Calculate model performance for the training data on a GloVe Model
print("Training performance:\n", base_train_gl)

#Calculating different metrics on validation data
base_val_gl = model_performance_classification_sklearn(base_gl,X_val_gl,y_val) # Calculate model performance for the validation data on a GloVe model.
print("Validation performance:\n",base_val_gl)

# Building the model

#Uncomment only one of the snippets related to fitting the model to the data

#base_wv = GradientBoostingClassifier(random_state = 42)
#base_wv = RandomForestClassifier(random_state=42)
base_st = DecisionTreeClassifier(random_state=42)

# Fitting on train data
base_st.fit(X_train_st, y_train) #Complete the code to fit the chosen model on the train data

plot_confusion_matrix(base_st,X_train_st,y_train) # Confusion matrix for the train data on our Transformer model.

plot_confusion_matrix(base_st,X_val_st,y_val) # Confusion matrix for the validation data on our Transformer model.

#Calculating different metrics on training data
base_train_st=model_performance_classification_sklearn(base_st,X_train_st,y_train) # Model performance for the training data on our Transformer Model.
print("Training performance:\n", base_train_st)

#Calculating different metrics on validation data
base_val_st = model_performance_classification_sklearn(base_st,X_val_st,y_val)  # Model performance for the validation data on our Transformer Model.
print("Validation performance:\n",base_val_st)

start = time.time()

# Choose the type of classifier.

#Uncomment only one of the snippets corrrsponding to the base model trained previously

#tuned_wv = GradientBoostingClassifier(random_state = 42)
#tuned_wv = RandomForestClassifier(random_state=42)
tuned_wv = DecisionTreeClassifier(random_state=42)


parameters = {
    'max_depth': np.arange(3,7),
    'min_samples_split': np.arange(5,12,2),
    'max_features': ['log2', 'sqrt', 0.2, 0.4]
}

# Run the grid search
grid_obj = GridSearchCV(tuned_wv, parameters, scoring='f1_weighted',cv=5,n_jobs=-1)
grid_obj = grid_obj.fit(X_train_wv, y_train)

end = time.time()
print("Time taken ",(end-start))

# Set the clf to the best combination of parameters
tuned_wv = grid_obj.best_estimator_

# Fit the best algorithm to the data.
tuned_wv.fit(X_train_wv, y_train)

plot_confusion_matrix(tuned_wv,X_train_wv,y_train)

plot_confusion_matrix(tuned_wv,X_val_wv,y_val)

#Calculating different metrics on training data
tuned_train_wv=model_performance_classification_sklearn(tuned_wv,X_train_wv,y_train)
print("Training performance:\n",tuned_train_wv)

#Calculating different metrics on validation data
tuned_val_wv = model_performance_classification_sklearn(tuned_wv,X_val_wv,y_val)
print("Validation performance:\n",tuned_val_wv)

start = time.time()

#Uncomment only one of the snippets corrrsponding to the base model trained previously

#tuned_wv = GradientBoostingClassifier(random_state = 42)
#tuned_wv = RandomForestClassifier(random_state=42)
tuned_gl = DecisionTreeClassifier(random_state=42)

parameters = {
    'max_depth': np.arange(3,7),
    'min_samples_split': np.arange(5,12,2),
    'max_features': ['log2', 'sqrt', 0.2, 0.4]
}

# Run the grid search
grid_obj = GridSearchCV(tuned_gl, parameters, scoring='f1_weighted',cv=5,n_jobs=-1) #Complete the code to pass the chosen model
grid_obj = grid_obj.fit(X_train_gl, y_train)

end = time.time()
print("Time taken ",(end-start))

# Set the clf to the best combination of parameters
tuned_gl = grid_obj.best_estimator_

# Fit the best algorithm to the data.
tuned_gl.fit(X_train_gl, y_train) # Fit the chosen model on the train data

plot_confusion_matrix(tuned_gl,X_train_gl, y_train) # Confusion matrix for the train data

plot_confusion_matrix(tuned_gl,X_val_gl,y_val) # Confusion matrix for the validation data

# Metrics on training data
tuned_train_gl=model_performance_classification_sklearn(tuned_gl,X_train_gl, y_train) # Model performance for the training data on GloVe model.
print("Training performance:\n",tuned_train_gl)

#Calculating different metrics on validation data
tuned_val_gl = model_performance_classification_sklearn(tuned_gl,X_val_gl,y_val) # Model performance for the validation data on GloVe model.
print("Validation performance:\n",tuned_val_gl)

start = time.time()

# Choose the type of classifier.

#Uncomment only one of the snippets corrrsponding to the base model trained previously

#tuned_wv = GradientBoostingClassifier(random_state = 42)
#tuned_wv = RandomForestClassifier(random_state=42)
tuned_st = DecisionTreeClassifier(random_state=42)

parameters = {
    'max_depth': np.arange(3,7),
    'min_samples_split': np.arange(5,12,2),
    'max_features': ['log2', 'sqrt', 0.2, 0.4]
}

# Run the grid search
grid_obj = GridSearchCV(tuned_st, parameters, scoring='f1_weighted',cv=5,n_jobs=-1) #Complete the code to pass the chosen model
grid_obj = grid_obj.fit(X_train_st, y_train)

end = time.time()
print("Time taken ",(end-start))

# Set the clf to the best combination of parameters
tuned_st = grid_obj.best_estimator_

# Fit the best algorithm to the data.
tuned_st.fit(X_train_st, y_train) #Complete the code to fit the chosen model on the train data

plot_confusion_matrix(tuned_st,X_train_st,y_train) #Complete the code to plot the confusion matrix for the train data

plot_confusion_matrix(tuned_st,X_val_st,y_val) #Complete the code to plot the confusion matrix for the validation data

# Metrics on training data
tuned_train_st=model_performance_classification_sklearn(tuned_st,X_train_st,y_train) #C Model performance for the training data
print("Training performance:\n",tuned_train_st)

# Metrics on validation data
tuned_val_st = model_performance_classification_sklearn(tuned_st,X_val_st,y_val) # Model performance for the validation data
print("Validation performance:\n",tuned_val_st)

#training performance comparison

models_train_comp_df = pd.concat(
    [base_train_wv.T,
     base_train_gl.T,
     base_train_st.T,
     tuned_train_wv.T,
     tuned_train_gl.T,
     tuned_train_st.T,
    ],axis=1
)

models_train_comp_df.columns = [
    "Base Model (Word2Vec)",
    "Base Model (GloVe)",
    "Base Model (Sentence Transformer)",
    "Tuned Model (Word2Vec)",
    "Tuned Model (GloVe)",
    "Tuned Model (Sentence Transformer)",
]

print("Training performance comparison:")
models_train_comp_df

# Validation performance comparison

models_val_comp_df = pd.concat(
    [base_val_wv.T,
     base_val_gl.T,
     base_val_st.T,
     tuned_val_wv.T,
     tuned_val_gl.T,
     tuned_val_st.T,
     ],axis=1
)

models_val_comp_df.columns = [
    "Base Model (Word2Vec)",
    "Base Model (GloVe)",
    "Base Model (Sentence Transformer)",
    "Tuned Model (Word2Vec)",
    "Tuned Model (GloVe)",
    "Tuned Model (Sentence Transformer)",
]

print("Validation performance comparison:")
models_val_comp_df

# Fit the best model to the test data.
tuned_wv.fit(X_test_wv, y_test) # Fit the chosen model on the test data.

plot_confusion_matrix(tuned_wv,X_test_wv,y_test) # Confusion matrix for the final model and test data.

# Calculating different metrics on test data
final_model_test = model_performance_classification_sklearn(tuned_wv,X_test_wv,y_test) # Final model's performance with the test data.
print("Test performance for the final model:\n",final_model_test)