first

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.json filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.9
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+COPY --chown=user ./requirements.txt requirements.txt
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY --chown=user . /app
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

api.py

@@ -0,0 +1,342 @@
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+import pandas as pd
+import numpy as np
+import tensorflow as tf
+from yahoo_fin.stock_info import get_data
+from sklearn.preprocessing import MinMaxScaler
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+from pytorch_forecasting import  TemporalFusionTransformer
+from bs4 import BeautifulSoup
+import requests
+import torch
+from llama_index.llms.groq import Groq
+from llama_index.core import StorageContext, load_index_from_storage
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+from dotenv import load_dotenv
+from llama_index.llms.huggingface_api import HuggingFaceInferenceAPI
+import os
+
+load_dotenv()
+
+embed_model = HuggingFaceEmbedding(model_name="BAAI/bge-small-en-v1.5")
+storage_context = StorageContext.from_defaults(persist_dir="rag_index")
+index = load_index_from_storage(storage_context, embed_model=embed_model)
+
+llm = HuggingFaceInferenceAPI(
+    model_name="HuggingFaceH4/zephyr-7b-alpha", token=os.getenv('HF_API')
+)
+
+query_engine = index.as_query_engine(llm=llm)
+
+MODEL_PATH = "lib/20_lstm_model.h5"
+model = tf.keras.models.load_model(MODEL_PATH)
+
+model_name_news= "mrm8488/distilroberta-finetuned-financial-news-sentiment-analysis"
+tokenizer =  AutoTokenizer.from_pretrained(model_name_news)
+sentiment_model = AutoModelForSequenceClassification.from_pretrained(model_name_news)
+
+best_model_path = 'lib/tft_pred.ckpt'
+
+best_tft = TemporalFusionTransformer.load_from_checkpoint(best_model_path)
+
+app = FastAPI()
+
+class TickerRequest(BaseModel):
+    ticker: str
+    start_date: str
+    end_date: str
+    interval: str = "1d"
+
+def fetch_and_process_ticker_data(ticker, start_date, end_date, interval="1d"):
+    df = pd.DataFrame()
+    try:
+        temp = get_data(ticker, start_date=start_date, end_date=end_date, index_as_date=True, interval=interval)
+        temp = temp.drop(columns="close")
+        temp["revenue"] = temp["adjclose"] * temp["volume"]
+        temp["daily_profit"] = temp["adjclose"] - temp["open"]
+        df = pd.concat([df, temp], axis=0)
+        df.to_csv("api_test.csv", index=False)  # Save locally for reference
+    except Exception as error:
+        raise HTTPException(status_code=500, detail=f"Error processing ticker {ticker}: {error}")
+    return df
+
+def ticker_encoded(df):
+    label_map = {'ATOM': 0, 'HBIO': 1, 'IBEX': 2, 'MYFW': 3, 'NATH': 4}
+
+    ticker_encoded = []
+
+    for i in df.iloc():
+
+        ticker_name = i['ticker']
+
+        encoded_ticker = label_map[ticker_name]
+
+        ticker_encoded.append(encoded_ticker)
+    df['ticker_encoded'] = ticker_encoded
+
+    return df
+
+def normalize(df):
+    price_scaler = MinMaxScaler()
+    volume_revenue_scaler = MinMaxScaler()
+    profit_scaler = MinMaxScaler()
+
+    df[["open", "high", "low", "adjclose"]] = price_scaler.fit_transform(df[["open", "high", "low", "adjclose"]])
+    df[["volume", "revenue"]] = volume_revenue_scaler.fit_transform(df[["volume", "revenue"]])
+    df[["daily_profit"]] = profit_scaler.fit_transform(df[["daily_profit"]])
+
+    return df, price_scaler
+
+def create_sequence(dataset):
+    sequences = []
+    labels = []
+    dates = []
+    stock = []
+
+    df_copy = dataset.drop(columns=["date"])
+
+    start_idx = 0
+    for stop_idx in range(20, len(dataset)):
+        set_ = set(dataset.iloc[start_idx:stop_idx]["ticker_encoded"].values)
+        target_day_ticker_class = dataset.iloc[stop_idx]["ticker_encoded"]
+
+        if len(set_) == 1 and target_day_ticker_class == list(set_)[0]:
+            sequences.append(df_copy.iloc[start_idx:stop_idx].values)
+            labels.append(df_copy.iloc[stop_idx][["open", "adjclose"]])
+            date_string = dataset.iloc[stop_idx]["date"].strftime('%Y-%m-%d')
+            dates.append(date_string)
+            stock.append(str(dataset.iloc[stop_idx]["ticker_encoded"]))
+
+        start_idx += 1
+
+    return np.array(sequences), np.array(labels), dates, stock
+
+def scaling_predictions(price_scaler,combined_dataset_prediction):
+
+    price_scaler.min_ = np.array([price_scaler.min_[0], price_scaler.min_[3]])
+
+    price_scaler.scale_ = np.array([price_scaler.scale_[0], price_scaler.scale_[3]])
+
+    combined_dataset_prediction_inverse =price_scaler.inverse_transform(combined_dataset_prediction)
+
+    return combined_dataset_prediction_inverse
+
+def storing_predictions(df,dates,stock,combined_dataset_prediction_inverse):
+
+    df['pred_open'] = np.nan
+
+    df['pred_closing'] = np.nan
+
+    for idx, row in df.iterrows():
+
+        current_row_date = row.date.strftime('%Y-%m-%d')
+
+        current_row_ticker = str(row.ticker_encoded)
+
+        for i in range(len(dates)):
+
+
+            if current_row_date == dates[i] and stock[i] == current_row_ticker:
+
+                opening_price = combined_dataset_prediction_inverse[i][0]
+                closing_price = combined_dataset_prediction_inverse[i][1]
+                df.loc[idx, 'pred_open'] = opening_price
+                df.loc[idx, 'pred_closing'] = closing_price
+
+                break
+    df = df.dropna(subset=['pred_open', 'pred_closing']).reset_index(drop=True)
+
+    return df
+
+def scrape_news(ticker_name):
+
+    columns = ['datatime', 'title','source', 'link','top_sentiment','sentiment_score']
+    df = pd.DataFrame(columns=columns)
+
+    for i in range (1,3):
+
+        url = f'https://markets.businessinsider.com/news/{ticker_name}-stock?p={i}'
+        response = requests.get(url)
+        html = response.text
+        soup = BeautifulSoup(html, 'lxml')
+
+        articles = soup.find_all('div',class_= 'latest-news__story')
+
+        for article in articles:
+            datatime = article.find('time', class_ = 'latest-news__date').get('datetime')
+
+            title = article.find('a', class_ = 'news-link').text
+
+            source = article.find('span', class_ = 'latest-news__source').text
+
+            link = article.find('a', class_ = 'news-link').get('href')
+
+            top_sentiment = ''
+
+            sentiment_score = 0
+
+            temp = pd.DataFrame([[datatime,title, source,link, top_sentiment,sentiment_score]], columns= df.columns)
+
+            df = pd.concat([temp,df], axis = 0)
+
+    return df
+
+def add_recent_news(main_df, news_df,lookback_days=10):
+    
+    news_df.drop(columns=['top_sentiment', 'sentiment_score'], inplace=True)
+
+    
+    main_df['date'] = pd.to_datetime(main_df['date'])
+    news_df['datatime'] = pd.to_datetime(news_df['datatime'])
+
+    
+    news_list = []
+    last_available_news = ''
+
+    for _, row in main_df.iterrows():
+        current_date = row['date']
+        current_ticker = row['ticker']
+        news_articles = ''
+
+        
+        for _, news_row in news_df.iterrows():
+            extracted_date = news_row['datatime']
+
+            
+            if (current_date - extracted_date).days <= lookback_days and extracted_date < current_date:
+                news_articles += news_row['title'] + " "  
+
+        
+        if not news_articles.strip():
+            for _, news_row in news_df[::-1].iterrows():
+                if news_row['datatime'] < current_date:
+                    news_articles = news_row['title']
+                    break
+
+        
+        last_available_news = news_articles.strip() or last_available_news
+        news_list.append(last_available_news)
+
+    
+    main_df['news'] = news_list
+
+    
+    return main_df
+
+def news_sentiment(df):
+
+    news_column_name = 'news'
+    texts = df[news_column_name].tolist()
+
+
+    inputs = tokenizer(texts, padding=True, truncation=True, return_tensors="pt")
+
+    with torch.no_grad():
+        outputs = sentiment_model(**inputs)
+
+    
+    logits = outputs.logits
+    probs = torch.softmax(logits, dim=-1)
+
+    
+    labels = ["negative", "neutral", "positive"]
+
+
+    predictions = torch.argmax(probs, dim=-1)
+
+    
+    df['predicted_sentiment'] = pd.Series([labels[pred] for pred in predictions], index=df[df[news_column_name].notna()].index)
+
+    sentiment_map = {
+    'positive': 1,
+    'neutral': 0,
+    'negative': -1
+    }
+
+
+    df['sentiment_score'] = df['predicted_sentiment'].map(sentiment_map)
+
+    df = df.drop(columns=['news'])
+
+    return df
+
+def get_tft_predictions(df):
+    for i in range(1, 21):
+        df[f'open_lag_{i}'] = df.groupby('ticker')['open'].shift(i)
+        df[f'adjclose_lag_{i}'] = df.groupby('ticker')['adjclose'].shift(i)
+
+    lag_columns = [f'open_lag_{i}' for i in range(1, 21)] + [f'adjclose_lag_{i}' for i in range(1, 21)]
+
+    df.dropna(subset=lag_columns, inplace=True)
+
+    predictions = best_tft.predict(df, mode="quantiles")
+
+    return predictions
+
+@app.post("/fetch-ticker-data/")
+async def fetch_ticker_data(request: TickerRequest):
+        try:
+            result_df = fetch_and_process_ticker_data(
+                ticker=request.ticker,
+                start_date=request.start_date,
+                end_date=request.end_date,
+                interval=request.interval
+            )
+            return result_df.to_dict(orient="records")
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/predict-prices/")
+async def predict_prices(request: TickerRequest):
+    try:
+        raw_data = fetch_and_process_ticker_data(
+            ticker=request.ticker,
+            start_date=request.start_date,
+            end_date=request.end_date,
+            interval=request.interval
+        )
+
+
+        raw_data = raw_data.tail(60)
+        raw_data= raw_data.reset_index()
+
+
+        raw_data.rename(columns={"index": "date"}, inplace=True)
+        raw_data = ticker_encoded(raw_data)
+
+        temp_df = raw_data.copy()  
+
+        normalized_data, scaler = normalize(raw_data)
+        normalized_data = normalized_data.drop(columns=['ticker'])
+
+        sequences, _, dates, stock = create_sequence(normalized_data)
+        combined_dataset_prediction = model.predict(sequences)
+        combined_dataset_prediction_inverse = scaling_predictions(scaler,combined_dataset_prediction)
+
+
+        lstm_pred_df=storing_predictions(temp_df,dates,stock,combined_dataset_prediction_inverse)
+        news_df = scrape_news(ticker_name = request.ticker)
+
+        combined_with_news_df = add_recent_news(lstm_pred_df,news_df)
+        sentiment_df = news_sentiment(combined_with_news_df)
+        
+        sentiment_df['time_idx'] = range(1, len(sentiment_df) + 1)
+
+        predicted_values = get_tft_predictions(sentiment_df)
+
+        final_pred_open_price = predicted_values[0].item()  
+        final_pred_closing_price = predicted_values[1].item() 
+
+        return {"open": final_pred_open_price, 'close': final_pred_closing_price}  
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+    
+
+@app.get("/query-rag/{user_query}")
+def query_rag(user_query:str):
+
+    response = query_engine.query(user_query)
+
+    return {'message':response}

lib/20_lstm_model.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdb810150ae9d87484ce4ccc4fa5434647820411b2400b41c6fe368a3fa12f7a
+size 422880

lib/tft_pred.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a38216d56d429e8f038efe3c1d83996d06b3e02b296352cf17e1e635c579371
+size 2885961

rag_index/default__vector_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b07b216dd34042722c022963768ab830d48d385c645623e46afc83b37a4745c0
+size 14374003

rag_index/docstore.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbf3547cf968b289a8fff77a16cb0143649df8424f45843822ad9c6853bf3d45
+size 7500231

rag_index/graph_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e0a77744010862225c69da83c585f4f8a42fd551b044ce530dbb1eb6e16742c
+size 18

rag_index/image__vector_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d17ed74c1649a438e518a8dc56a7772913dfe1ea7a7605bce069c63872431455
+size 72

rag_index/index_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0128597297ccb9b86477e4805882f2501c988031e30cc651d857ad3a5a3b870c
+size 133807

requirements.txt

@@ -0,0 +1,18 @@
+fastapi
+pydantic
+pandas
+numpy
+tensorflow==2.13.0
+yahoo-fin
+scikit-learn
+transformers==4.39.2
+pytorch_forecasting
+beautiful
+requests
+torch
+llama-index
+llama-index-core
+llama-index-embeddings-huggingface
+dotenv
+llama-index-llms-huggingface-api
+keras==3.2.1