update

retrieve.py

@@ -85,240 +85,4 @@ def search(
             f"Search failed for query '{query_text[:100]}...': {type(e).__name__}: {e}",
             exc_info=True,
         )
-        raise
-    
-def search(
-    query_text: str,
-    cohere_client: cohere.ClientV2,
-    qdrant_client: QdrantClient,
-    collection_name: str,
-    top_k: int = 5,
-) -> List[Dict[str, Any]]:
-    """
-    Convert query to embedding and retrieve top-K relevant chunks.
-
-    Args:
-        query_text: User's search query (non-empty, ≤1000 chars)
-        top_k: Number of results to return (1-100)
-
-    Returns:
-        List of search results with id, score, and payload
-    """
-    # Validate inputs
-    if not query_text or not query_text.strip():
-        raise ValueError("Query text must be non-empty")
-    query_text = query_text.strip()
-    if len(query_text) > 1000:
-        raise ValueError("Query text must be ≤ 1000 characters")
-    if top_k < 1 or top_k > 100:
-        raise ValueError("top_k must be between 1 and 100")
-
-    logger.info(f"Embedding query: '{query_text[:100]}...' (top_k={top_k})")
-    start_time = time.time()
-
-    # Generate query embedding with retry
-    try:
-        embedding = utils.retry_with_backoff(
-            lambda: embed_query(query_text, cohere_client),
-            max_retries=3,
-            base_delay=1.0,
-            max_delay=10.0,
-        )
-        embed_time = time.time() - start_time
-        logger.debug(
-            f"Generated embedding in {embed_time:.2f}s, dimension: {len(embedding)}"
-        )
-    except Exception as e:
-        logger.error(f"Failed to embed query: {e}")
-        raise
-
-    # Search Qdrant with retry
-    try:
-        search_start = time.time()
-        response = utils.retry_with_backoff(
-            lambda: qdrant_client.query_points(
-                collection_name=collection_name,
-                query=embedding,
-                limit=top_k,
-                with_payload=True,
-                with_vectors=False,
-            ),
-            max_retries=3,
-            base_delay=1.0,
-            max_delay=10.0,
-        )
-        results = response.points
-        search_time = time.time() - search_start
-        logger.info(
-            f"Search completed in {search_time:.2f}s, returned {len(results)} results"
-        )
-    except Exception as e:
-        logger.error(f"Search failed: {e}")
-        raise APIError(f"Qdrant search failed: {e}")
-
-    # Format results
-    formatted = []
-    for result in results:
-        formatted.append(
-            {
-                "id": str(result.id),
-                "score": float(result.score),
-                "payload": result.payload,
-            }
-        )
-
-    total_time = time.time() - start_time
-    logger.info(f"Total query time: {total_time:.2f}s")
-
-    return formatted
-
-
-def format_results(
-    results: List[Dict[str, Any]], query: str, latency_ms: int
-) -> Dict[str, Any]:
-    """Format search results into JSON output structure."""
-    output = {
-        "query": query,
-        "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
-        "results": results,
-        "metadata": {
-            "total_results": len(results),
-            "collection": None,  # Will be filled by main
-            "latency_ms": latency_ms,
-        },
-    }
-    return output
-
-
-def main() -> int:
-    """CLI entrypoint for retrieval."""
-    parser = argparse.ArgumentParser(
-        description="Retrieve relevant chunks from Qdrant using Cohere embeddings"
-    )
-    parser.add_argument("--query", type=str, help="Search query text")
-    parser.add_argument(
-        "--top-k", type=int, default=5, help="Number of results to return (default: 5)"
-    )
-    parser.add_argument("--output", type=str, help="Output file path (default: stdout)")
-    parser.add_argument(
-        "--config",
-        type=str,
-        default=".env",
-        help="Path to .env config file (default: .env)",
-    )
-    parser.add_argument(
-        "--validate-metadata",
-        action="store_true",
-        help="Run metadata validation on search results (requires --query)",
-    )
-
-    args = parser.parse_args()
-
-    # Setup logging
-    log_file = "retrieve.log"
-    setup_logging(log_file=log_file, console_level="INFO")
-    logger.info("=== Retrieval Pipeline Started ===")
-
-    try:
-        # Load config
-        logger.info(f"Loading config from {args.config}")
-        cfg = config.get_config()
-        validate_config(cfg)
-
-        # Initialize clients
-        logger.info("Initializing Cohere and Qdrant clients")
-        cohere_client, qdrant_client = init_clients(cfg)
-
-        # Check collection
-        collection_name = cfg["qdrant_collection"]
-        logger.info(f"Checking collection '{collection_name}'")
-        coll_info = check_collection(qdrant_client, collection_name)
-        logger.info(
-            f"Collection OK: vector_size={coll_info['vector_size']}, points={coll_info['points_count']}"
-        )
-
-        # Validate query argument
-        if not args.query:
-            parser.error("--query is required")
-
-        # Perform search
-        results = search(
-            query_text=args.query,
-            cohere_client=cohere_client,
-            qdrant_client=qdrant_client,
-            collection_name=collection_name,
-            top_k=args.top_k,
-        )
-
-        # Perform metadata validation if requested
-        metadata_validation = None
-        if args.validate_metadata:
-            completeness = validate_metadata_completeness(results)
-            sequencing = validate_chunk_sequencing(results)
-            metadata_validation = {
-                "completeness_pct": round(completeness, 2),
-                "sequencing_valid": sequencing,
-                "pass": completeness >= 98.0 and sequencing,
-            }
-            logger.info(f"Metadata completeness: {completeness:.1f}%")
-            logger.info(f"Chunk sequencing: {'VALID' if sequencing else 'INVALID'}")
-            logger.info(
-                f"Validation result: {'PASS' if metadata_validation['pass'] else 'FAIL'}"
-            )
-
-        # Format output
-        output = {
-            "query": args.query,
-            "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
-            "results": results,
-            "metadata": {
-                "total_results": len(results),
-                "collection": collection_name,
-                "vector_size": coll_info["vector_size"],
-                "points_count": coll_info["points_count"],
-            },
-        }
-
-        if metadata_validation:
-            output["metadata_validation"] = metadata_validation
-
-        # Output JSON
-        json_output = json.dumps(output, indent=2)
-        if args.output:
-            with open(args.output, "w") as f:
-                f.write(json_output)
-            logger.info(f"Results written to {args.output}")
-        else:
-            print(json_output)
-
-        logger.info("=== Retrieval Pipeline Completed Successfully ===")
-        return 0
-
-    except ValueError as ve:
-        logger.error(f"Validation error: {ve}")
-        print(f"ERROR: {ve}", file=sys.stderr)
-        return 2
-    except ConfigurationError as ce:
-        logger.error(f"Configuration error: {ce}")
-        print(f"ERROR: {ce}", file=sys.stderr)
-        return 1
-    except CollectionNotFoundError as cnfe:
-        logger.error(f"Collection error: {cnfe}")
-        print(f"ERROR: {cnfe}", file=sys.stderr)
-        return 1
-    except DimensionMismatchError as dme:
-        logger.error(f"Dimension error: {dme}")
-        print(f"ERROR: {dme}", file=sys.stderr)
-        return 1
-    except APIError as api_err:
-        logger.error(f"API error: {api_err}")
-        print(f"ERROR: {api_err}", file=sys.stderr)
-        return 1
-    except Exception as e:
-        logger.exception(f"Unexpected error: {e}")
-        print(f"ERROR: Unexpected error: {e}", file=sys.stderr)
-        return 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())
+        raise