app.py

import gradio as gr
from transformers import pipeline
# from embedding_gen import load_skills_from_date, visualize3D
import numpy as np
import pickle

# token_skill_classifier = pipeline(model="jjzha/jobbert_skill_extraction", aggregation_strategy="first")
# token_knowledge_classifier = pipeline(model="jjzha/jobbert_knowledge_extraction")
# token_knowledge_classifier = pipeline(model="Robzy/jobbert_knowledge_extraction")


examples = [
        "High proficiency in Python and AI/ML frameworks, i.e. Pytorch.",
        "Experience with Unreal and/or Unity and/or native IOS/Android 3D development",
        ]


def aggregate_span(results):
    new_results = []
    current_result = results[0]

    for result in results[1:]:
        if result["start"] == current_result["end"] + 1:
            current_result["word"] += " " + result["word"]
            current_result["end"] = result["end"]
        else:
            new_results.append(current_result)
            current_result = result

    new_results.append(current_result)

    return new_results

# def ner(text):


#     output_knowledge = token_knowledge_classifier(text)
#     for result in output_knowledge:
#         if result.get("entity_group"):
#             result["entity"] = "Knowledge"
#             del result["entity_group"]

#     if len(output_knowledge) > 0:
#         output_knowledge = aggregate_span(output_knowledge)

#     return {"text": text, "entities": output_knowledge}

### Visualisation 3D

import os

def load_skills_from_date(base_folder, date):
    date_folder = os.path.join(base_folder, date)
    all_skills = set()  # To ensure unique skills
    if os.path.exists(date_folder) and os.path.isdir(date_folder):
        for file_name in os.listdir(date_folder):
            file_path = os.path.join(date_folder, file_name)
            if file_name.endswith(".txt"):
                with open(file_path, 'r', encoding='utf-8') as f:
                    all_skills.update(line.strip() for line in f if line.strip())
    return list(all_skills)

def visualize3D(reduced_embeddings, labels, skills, n_clusters, output_folder, date):
    
    fig = px.scatter_3d(
        x=reduced_embeddings[:, 0],
        y=reduced_embeddings[:, 1],
        z=reduced_embeddings[:, 2],
        color=labels,
        text=skills,
        title=f"KMeans Clustering with {n_clusters} Clusters ({date})"
    )
    
    # Save the clustered plot
    # os.makedirs(output_folder, exist_ok=True)
    # plot_path = os.path.join(output_folder, f"{date}_3D_clustering.html")
    # fig.write_html(plot_path)
    # print(f"3D clustered plot saved at {plot_path}")
    
    # fig.show()
    return fig



import plotly.express as px
import numpy as np

specific_date = "03-01-2024"  # Example date folder to process
skills = load_skills_from_date('./tags', specific_date)
embeddings = np.load(f"./vectorstore/{specific_date}_embeddings.npy")
with open(f"./vectorstore/{specific_date}_metadata.pkl", "rb") as f:
    metadata =   pickle.load(f)
labels, skills = metadata["labels"], metadata["skills"]
fig = visualize3D(embeddings, labels, skills, n_clusters=5, output_folder="./plots", date=specific_date)
fig.update_layout(
     height=900
)

with gr.Blocks() as demo:
    
    gr.Markdown("# 3D Visualization of Skills in ML Job Postings", elem_id="title")
    # gr.Markdown("Embedding visualisation of sought skills in ML job posting in Stockholm, Sweden on LinkedIn")
    gr.Plot(fig)

    

demo.launch(ssr_mode=False)