Going online (first public version)

.bandit.yml

@@ -0,0 +1,5 @@
+skips:
+- B101
+- B311
+- B320
+- B410

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+ignore = E501  # line too long
+exclude = .git,__pycache__,docs,.github,build,dist

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: D4Vinci

.github/ISSUE_TEMPLATE/01-bug_report.yml

@@ -0,0 +1,82 @@
+name: Bug report
+description: Create a bug report to help us address errors in the repository
+labels: [bug]
+body:
+  - type: checkboxes
+    attributes:
+      label: Have you searched if there an existing issue for this?
+      description: Please search [existing issues](https://github.com/D4Vinci/Scrapling/labels/bug).
+      options:
+        - label: I have searched the existing issues
+          required: true
+
+  - type: input
+    attributes:
+      label: "Python version (python --version)"
+      placeholder: "Python 3.8"
+    validations:
+      required: true
+
+  - type: input
+    attributes:
+      label: "Scrapling version (scrapling.__version__)"
+      placeholder: "0.1"
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: "Dependencies version (pip3 freeze)"
+      description: >
+        This is the output of the command `pip3 freeze --all`. Note that the
+        actual output might be different as compared to the placeholder text.
+      placeholder: |
+        cssselect==1.2.0
+        lxml==5.3.0
+        orjson==3.10.7
+        ...
+    validations:
+      required: true
+
+  - type: input
+    attributes:
+      label: "What's your operating system?"
+      placeholder: "Windows 10"
+    validations:
+      required: true
+
+  - type: dropdown
+    attributes:
+      label: 'Are you using a separate virtual environment?'
+      description: "Please pay attention to this question"
+      options:
+        - No
+        - Yes
+      default: 0
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: "Expected behavior"
+      description: "Describe the behavior you expect. May include images or videos."
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: "Actual behavior (Remember to use `debug` parameter)"
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Steps To Reproduce
+      description: Steps to reproduce the behavior.
+      placeholder: |
+        1. In this environment...
+        2. With this config...
+        3. Run '...'
+        4. See error...
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/02-feature_request.yml

@@ -0,0 +1,19 @@
+name: Feature request
+description: Suggest features, propose improvements, discuss new ideas.
+labels: [enhancement]
+body:
+  - type: checkboxes
+    attributes:
+      label: Have you searched if there an existing feature request for this?
+      description: Please search [existing requests](https://github.com/D4Vinci/Scrapling/labels/enhancement).
+      options:
+        - label: I have searched the existing requests
+          required: true
+
+  - type: textarea
+    attributes:
+      label: "Feature description"
+      description: >
+        This could include new topics or improving any existing features/implementations.
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/03-other.yml

@@ -0,0 +1,19 @@
+name: Other
+description: Use this for any other issues. PLEASE do not create blank issues
+labels: ["awaiting triage"]
+body:
+  - type: textarea
+    id: issuedescription
+    attributes:
+      label: What would you like to share?
+      description: Provide a clear and concise explanation of your issue.
+    validations:
+      required: true
+
+  - type: textarea
+    id: extrainfo
+    attributes:
+      label: Additional information
+      description: Is there anything else we should know about this issue?
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: false

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,53 @@
+<!--
+  You are amazing! Thanks for contributing to Scrapling!
+  Please, DO NOT DELETE ANY TEXT from this template! (unless instructed).
+-->
+
+## Proposed change
+<!--
+  Describe the big picture of your changes here to communicate to the
+  maintainers why we should accept this pull request. If it fixes a bug
+  or resolves a feature request, be sure to link to that issue in the
+  additional information section.
+-->
+
+
+### Type of change:
+<!--
+  What type of change does your PR introduce to Scrapling?
+  NOTE: Please, check at least 1 box!
+  If your PR requires multiple boxes to be checked, you'll most likely need to
+  split it into multiple PRs. This makes things easier and faster to code review.
+-->
+
+
+
+- [ ] Dependency upgrade
+- [ ] Bugfix (non-breaking change which fixes an issue)
+- [ ] New integration (thank you!)
+- [ ] New feature (which adds functionality to an existing integration)
+- [ ] Deprecation (breaking change to happen in the future)
+- [ ] Breaking change (fix/feature causing existing functionality to break)
+- [ ] Code quality improvements to existing code or addition of tests
+- [ ] Add or change doctests? -- Note: Please avoid changing both code and tests in a single pull request.
+- [ ] Documentation change?
+
+### Additional information
+<!--
+  Details are important, and help maintainers processing your PR.
+  Please be sure to fill out additional details, if applicable.
+-->
+
+- This PR fixes or closes issue: fixes #
+- This PR is related to issue: 
+- Link to documentation pull request: **
+
+### Checklist:
+* [ ] I have read [CONTRIBUTING.md](/CONTRIBUTING.md).
+* [ ] This pull request is all my own work -- I have not plagiarized.
+* [ ] I know that pull requests will not be merged if they fail the automated tests.
+* [ ] All new Python files are placed inside an existing directory.
+* [ ] All filenames are in all lowercase characters with no spaces or dashes.
+* [ ] All functions and variable names follow Python naming conventions.
+* [ ] All function parameters and return values are annotated with Python [type hints](https://docs.python.org/3/library/typing.html).
+* [ ] All functions have doc-strings.

.github/workflows/publish.yml

@@ -0,0 +1,31 @@
+name: Publish Python 🐍 distributions 📦 to PyPI
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  build-n-publish:
+    name: Build and publish Python 🐍 distributions 📦 to PyPI
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"  # Latest available Python version
+
+      - name: Upgrade pip
+        run: python3 -m pip install --upgrade pip
+
+      - name: Install build
+        run: python3 -m pip install --upgrade build twine setuptools
+
+      - name: Build a binary wheel and a source tarball
+        run: python3 -m build --sdist --wheel --outdir dist/
+
+      - name: Publish distribution 📦 to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1.10.3
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/tests.yml

@@ -0,0 +1,48 @@
+name: Tests
+on: [push, pull_request]
+
+concurrency:
+  group: ${{github.workflow}}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - python-version: "3.6"
+          env:
+            TOXENV: py
+        - python-version: "3.7"
+          env:
+            TOXENV: py
+        - python-version: "3.8"
+          env:
+            TOXENV: py
+        - python-version: "3.9"
+          env:
+            TOXENV: py
+        - python-version: "3.10"
+          env:
+            TOXENV: py
+        - python-version: "3.11"
+          env:
+            TOXENV: py
+        - python-version: "3.12"
+          env:
+            TOXENV: py
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Run tests
+      env: ${{ matrix.env }}
+      run: |
+        pip install -U tox
+        tox

.gitignore

@@ -1,128 +1,25 @@
-# Byte-compiled / optimized / DLL files
+# cached files
 __pycache__/
 *.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
 .cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
-.pdm.toml
-.pdm-python
-.pdm-build/
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
+.DS_Store
+*~
+.*.sw[po]
+.build
+.ve
+.env
+.pytest
+.benchmarks
+.bootstrap
+.appveyor.token
+*.bak
 
-# SageMath parsed files
-*.sage.py
+# installation package
+*.egg-info/
+dist/
+build/
 
-# Environments
-.env
+# environments
 .venv
 env/
 venv/
@@ -130,33 +27,73 @@ ENV/
 env.bak/
 venv.bak/
 
-# Spyder project settings
-.spyderproject
-.spyproject
+# C extensions
+*.so
+
+# pycharm
+.idea/
 
-# Rope project settings
-.ropeproject
+# vscode
+*.code-workspace
 
-# mkdocs documentation
-/site
+# Packages
+*.egg
+*.egg-info
+dist
+build
+eggs
+.eggs
+parts
+bin
+var
+sdist
+wheelhouse
+develop-eggs
+.installed.cfg
+lib
+lib64
+venv*/
+.venv*/
+pyvenv*/
+pip-wheel-metadata/
+poetry.lock
+
+# Installer logs
+pip-log.txt
 
 # mypy
 .mypy_cache/
 .dmypy.json
 dmypy.json
+mypy.ini
+
+# test caches
+.tox/
+.pytest_cache/
+.coverage
+htmlcov
+report.xml
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
 
-# Pyre type checker
-.pyre/
+# Buildout
+.mr.developer.cfg
 
-# pytype static type analyzer
-.pytype/
+# IDE project files
+.project
+.pydevproject
+.idea
+*.iml
+*.komodoproject
 
-# Cython debug symbols
-cython_debug/
+# Complexity
+output/*.html
+output/*/index.html
 
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+# Sphinx
+docs/_build
+public/
+web/

.pre-commit-config.yaml

@@ -0,0 +1,14 @@
+repos:
+- repo: https://github.com/PyCQA/bandit
+  rev: 1.7.8
+  hooks:
+  - id: bandit
+    args: [-r, -c, .bandit.yml]
+- repo: https://github.com/PyCQA/flake8
+  rev: 7.0.0
+  hooks:
+  - id: flake8
+- repo: https://github.com/pycqa/isort
+  rev: 5.13.2
+  hooks:
+  - id: isort

CONTRIBUTING.md

@@ -0,0 +1,30 @@
+# Contributing to Scrapling
+Everybody is invited and welcome to contribute to Scrapling. Smaller changes have a better chance to get included in a timely manner. Adding unit tests for new features or test cases for bugs you've fixed help us to ensure that the Pull Request (PR) is fine.
+
+There is a lot to do...
+- If you are not a developer perhaps you would like to help with the [documentation](/docs)?
+- If you are a developer, most of the features I'm planning to add in the future are moved to [roadmap file](/ROADMAP.md) so consider reading it.
+
+Scrapling includes a comprehensive test suite which can be executed with pytest:
+```bash
+$ pytest
+=============================== test session starts ===============================
+platform darwin -- Python 3.12.7, pytest-8.3.3, pluggy-1.5.0
+rootdir: /<some_where>/Scrapling
+configfile: pytest.ini
+plugins: cov-5.0.0, anyio-4.6.0
+collected 16 items
+
+tests/test_all_functions.py ................          [100%]
+
+=============================== 16 passed in 0.22s ================================
+```
+Also, consider setting `debug` to `True` while initializing the Adaptor object so it's easier to know what's happening in the background.
+
+### The process is straight-forward.
+
+ - Read [How to get faster PR reviews](https://github.com/kubernetes/community/blob/master/contributors/guide/pull-requests.md#best-practices-for-faster-reviews) by Kubernetes (but skip step 0 and 1)
+ - Fork Scrapling [git repository](https://github.com/D4Vinci/Scrapling).
+ - Make your changes.
+ - Ensure tests work.
+ - Create a Pull Request against the [**dev**](https://github.com/D4Vinci/Scraplin/tree/dev) branch of Scrapling.

MANIFEST.in

@@ -0,0 +1,5 @@
+include LICENSE
+include scrapling/py.typed
+
+recursive-exclude * __pycache__
+recursive-exclude * *.py[co]

README.md

@@ -0,0 +1,434 @@
+# 🕷️ Scrapling: Lightning-Fast, Adaptive Web Scraping for Python
+[![PyPI version](https://badge.fury.io/py/scrapling.svg)](https://badge.fury.io/py/scrapling) [![Supported Python versions](https://img.shields.io/pypi/pyversions/scrapling.svg)](https://pypi.org/project/scrapling/) [![License](https://img.shields.io/badge/License-BSD--3-blue.svg)](https://opensource.org/licenses/BSD-3-Clause)
+
+Dealing with failing web scrapers due to website changes? Meet Scrapling.
+
+Scrapling is a high-performance, intelligent web scraping library for Python that automatically adapts to website changes while significantly outperforming popular alternatives. Whether you're a beginner or an expert, Scrapling provides powerful features while maintaining simplicity.
+
+```python
+from scrapling import Adaptor
+
+# Scrape data that survives website changes
+page = Adaptor(html, auto_match=True)
+products = page.css('.product', auto_save=True)
+# Later, even if selectors change:
+products = page.css('.product', auto_match=True)  # Still finds them!
+```
+
+## Key Features
+
+### Adaptive Scraping
+- 🔄 **Smart Element Tracking**: Locate previously identified elements after website structure changes, using an intelligent similarity system and integrated storage.
+- 🎯 **Flexible Querying**: Use CSS selectors, XPath, text search, or regex - chain them however you want!
+- 🔍 **Find Similar Elements**: Automatically locate elements similar to the element you want on the page (Ex: other products like the product you found on the page).
+- 🧠 **Smart Content Scraping**: Extract data from multiple websites without specific selectors using its powerful features.
+
+### Performance
+- 🚀 **Lightning Fast**: Built from the ground up with performance in mind, outperforming most popular Python scraping libraries (outperforming BeautifulSoup by up to 237x in our tests).
+- 🔋 **Memory Efficient**: Optimized data structures for minimal memory footprint.
+- ⚡ **Fast JSON serialization**: 10x faster JSON serialization than the standard json library with more options.
+
+### Developing Experience
+- 🛠️ **Powerful Navigation API**: Traverse the DOM tree easily in all directions and get the info you want (parent, ancestors, sibling, children, next/previous element, and more).
+- 🧬 **Rich Text Processing**: All strings have built-in methods for regex matching, cleaning, and more. All elements' attributes are read-only dictionaries that are faster than standard dictionaries with added methods.
+- 📝 **Automatic Selector Generation**: Create robust CSS/XPath selectors for any element.
+- 🔌 **Scrapy-Compatible API**: Familiar methods and similar pseudo-elements for Scrapy users.
+- 📘 **Type hints**: Complete type coverage for better IDE support and fewer bugs.
+
+## Getting Started
+
+Let's walk through a basic example that demonstrates small group of Scrapling's core features:
+
+```python
+import requests
+from scrapling import Adaptor
+
+# Fetch a web page
+url = 'https://quotes.toscrape.com/'
+response = requests.get(url)
+
+# Create an Adaptor instance
+page = Adaptor(response.text, url=url)
+# Get all strings in the full page
+page.get_all_text(ignore_tags=('script', 'style'))
+
+# Get all quotes, any of these methods will return a list of strings (TextHandlers)
+quotes = page.css('.quote .text::text')  # CSS selector
+quotes = page.xpath('//span[@class="text"]/text()')  # XPath
+quotes = page.css('.quote').css('.text::text')  # Chained selectors
+quotes = [element.text for element in page.css('.quote').css('.text')]  # Slower than bulk query above
+
+# Get the first quote element
+quote = page.css('.quote').first  # or [0] or .get()
+
+# Working with elements
+quote.html_content  # Inner HTML
+quote.prettify()  # Prettified version of Inner HTML
+quote.attrib  # Element attributes
+quote.path  # DOM path to element (List)
+```
+To keep it simple, all methods can be chained on top of each other as long as you are chaining methods that return an element (It's called an `Adaptor` object) or a List of Adaptors (It's called `Adaptors` object)
+
+### Installation
+Scrapling is a breeze to get started with - We only require at least Python 3.6 to work and the rest of the requirements are installed automatically with the package.
+```bash
+# Using pip
+pip install scrapling
+
+# Or the latest from GitHub
+pip install git+https://github.com/D4Vinci/Scrapling.git@master
+```
+
+## Performance
+
+Scrapling isn't just powerful - it's also blazing fast. Scrapling implements many best practices, design patterns, and numerous optimizations to save fractions of seconds. All of that while focusing exclusively on parsing HTML documents.
+Here are benchmarks comparing Scrapling to popular Python libraries in two tests. 
+
+### Text Extraction Speed Test (5000 nested elements).
+
+| # |      Library      | Time (ms) | vs Scrapling | 
+|---|:-----------------:|:---------:|:------------:|
+| 1 |     Scrapling     |   5.44    |     1.0x     |
+| 2 |   Parsel/Scrapy   |   5.53    |    1.017x    |
+| 3 |     Raw Lxml      |   6.76    |    1.243x    |
+| 4 |      PyQuery      |   21.96   |    4.037x    |
+| 5 |    Selectolax     |   67.12   |   12.338x    |
+| 6 |   BS4 with Lxml   |  1307.03  |   240.263x   |
+| 7 |  MechanicalSoup   |  1322.64  |   243.132x   |
+| 8 | BS4 with html5lib |  3373.75  |   620.175x   |
+
+As you see, Scrapling is on par with Scrapy and slightly faster than Lxml which both libraries are built on top of. These are the closest results to Scrapling. PyQuery is also built on top of Lxml but still, Scrapling is 4 times faster.
+
+### Extraction By Text Speed Test
+
+|   Library   | Time (ms) | vs Scrapling |
+|:-----------:|:---------:|:------------:|
+|  Scrapling  |   2.51    |     1.0x     |
+| AutoScraper |   11.41   |    4.546x    |
+
+Scrapling can find elements with more methods and it returns full element `Adaptor` objects not only the text like AutoScraper. So, to make this test fair, both libraries will extract an element with text, find similar elements, and then extract the text content for all of them. As you see, Scrapling is still 4.5 times faster at same task.
+
+> All benchmarks' results are an average of 100 runs. See our [benchmarks.py](/benchmarks.py) for methodology and to run your comparisons.
+
+## Advanced Features
+### Smart Navigation
+```python
+>>> quote.tag
+'div'
+
+>>> quote.parent
+<data='<div class="col-md-8"> <div class="quote...' parent='<div class="row"> <div class="col-md-8">...'>
+
+>>> quote.parent.tag
+'div'
+
+>>> quote.children
+[<data='<span class="text" itemprop="text">“The...' parent='<div class="quote" itemscope itemtype="h...'>,
+ <data='<span>by <small class="author" itemprop=...' parent='<div class="quote" itemscope itemtype="h...'>,
+ <data='<div class="tags"> Tags: <meta class="ke...' parent='<div class="quote" itemscope itemtype="h...'>]
+
+>>> quote.siblings
+[<data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>,
+ <data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>,
+ <data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>,
+...]
+
+>>> quote.next  # gets the next element, the same logic applies to `quote.previous`
+<data='<div class="quote" itemscope itemtype="h...' parent='<div class="col-md-8"> <div class="quote...'>
+
+>>> quote.children.css(".author::text")
+['Albert Einstein']
+
+>>> quote.has_class('quote')
+True
+
+# Generate new selectors for any element
+>>> quote.css_selector
+'body > div > div:nth-of-type(2) > div > div'
+
+# Test these selectors on your favorite browser or reuse them again in the library in other methods!
+>>> quote.xpath_selector
+'//body/div/div[2]/div/div'
+```
+If your case needs more than the element's parent, you can iterate over the whole ancestors' tree of any element like below
+```python
+for ancestor in quote.iterancestors():
+    # do something with it...
+```
+You can search for a specific ancestor of an element that satisfies a function, all you need to do is to pass a function that takes an `Adaptor` object as an argument and return `True` if the condition satisfies or `False` otherwise like below:
+```python
+>>> quote.find_ancestor(lambda ancestor: ancestor.has_class('row'))
+<data='<div class="row"> <div class="col-md-8">...' parent='<div class="container"> <div class="row...'>
+```
+
+### Content-based Selection & Finding Similar Elements
+You can select elements by their text content in multiple ways, here's a full example on another website:
+```python
+>>> response = requests.get('https://books.toscrape.com/index.html')
+
+>>> page = Adaptor(response.text, url=response.url)
+
+>>> page.find_by_text('Tipping the Velvet')  # Find the first element that its text fully matches this text
+<data='<a href="catalogue/tipping-the-velvet_99...' parent='<h3><a href="catalogue/tipping-the-velve...'>
+
+>>> page.find_by_text('Tipping the Velvet', first_match=False)  # Get all matches if there are more
+[<data='<a href="catalogue/tipping-the-velvet_99...' parent='<h3><a href="catalogue/tipping-the-velve...'>]
+
+>>> page.find_by_regex(r'£[\d\.]+')  # Get the first element that its text content matches my price regex
+<data='<p class="price_color">£51.77</p>' parent='<div class="product_price"> <p class="pr...'>
+
+>>> page.find_by_regex(r'£[\d\.]+', first_match=False)  # Get all elements that matches my price regex
+[<data='<p class="price_color">£51.77</p>' parent='<div class="product_price"> <p class="pr...'>,
+ <data='<p class="price_color">£53.74</p>' parent='<div class="product_price"> <p class="pr...'>,
+ <data='<p class="price_color">£50.10</p>' parent='<div class="product_price"> <p class="pr...'>,
+ <data='<p class="price_color">£47.82</p>' parent='<div class="product_price"> <p class="pr...'>,
+ ...]
+```
+Find all elements that are similar to the current element in location and attributes
+```python
+# For this case, ignore the 'title' attribute while matching
+>>> page.find_by_text('Tipping the Velvet').find_similar(ignore_attributes=['title'])
+[<data='<a href="catalogue/a-light-in-the-attic_...' parent='<h3><a href="catalogue/a-light-in-the-at...'>,
+ <data='<a href="catalogue/soumission_998/index....' parent='<h3><a href="catalogue/soumission_998/in...'>,
+ <data='<a href="catalogue/sharp-objects_997/ind...' parent='<h3><a href="catalogue/sharp-objects_997...'>,
+...]
+
+# You will notice that the number of elements is 19 not 20 because the current element is not included.
+>>> len(page.find_by_text('Tipping the Velvet').find_similar(ignore_attributes=['title']))
+19
+
+# Get the `href` attribute from all similar elements
+>>> [element.attrib['href'] for element in page.find_by_text('Tipping the Velvet').find_similar(ignore_attributes=['title'])]
+['catalogue/a-light-in-the-attic_1000/index.html',
+ 'catalogue/soumission_998/index.html',
+ 'catalogue/sharp-objects_997/index.html',
+ ...]
+```
+To increase the complexity a little bit, let's say we want to get all books' data using that element as a starting point for some reason
+```python
+>>> for product in page.find_by_text('Tipping the Velvet').parent.parent.find_similar():
+        print({
+            "name": product.css('h3 a::text')[0],
+            "price": product.css('.price_color')[0].re_first(r'[\d\.]+'),
+            "stock": product.css('.availability::text')[-1].clean()
+        })
+{'name': 'A Light in the ...', 'price': '51.77', 'stock': 'In stock'}
+{'name': 'Soumission', 'price': '50.10', 'stock': 'In stock'}
+{'name': 'Sharp Objects', 'price': '47.82', 'stock': 'In stock'}
+...
+```
+The [documentation](/docs/Examples) will provide more advanced examples.
+
+### Handling Structural Changes
+> Because [the internet archive](https://web.archive.org/) is down at the time of writing this, I can't use real websites as examples even though I tested that before (I mean browsing an old version of a website and then counting the current version of the website as structural changes)
+
+Let's say you are scraping a page with a structure like this:
+```html
+<div class="container">
+    <section class="products">
+        <article class="product" id="p1">
+            <h3>Product 1</h3>
+            <p class="description">Description 1</p>
+        </article>
+        <article class="product" id="p2">
+            <h3>Product 2</h3>
+            <p class="description">Description 2</p>
+        </article>
+    </section>
+</div>
+```
+and you want to scrape the first product, the one with the `p1` ID. You will probably write a selector like this
+```python
+page.css('#p1')
+```
+When website owners implement structural changes like
+```html
+<div class="new-container">
+    <div class="product-wrapper">
+        <section class="products">
+            <article class="product new-class" data-id="p1">
+                <div class="product-info">
+                    <h3>Product 1</h3>
+                    <p class="new-description">Description 1</p>
+                </div>
+            </article>
+            <article class="product new-class" data-id="p2">
+                <div class="product-info">
+                    <h3>Product 2</h3>
+                    <p class="new-description">Description 2</p>
+                </div>
+            </article>
+        </section>
+    </div>
+</div>
+```
+The selector will no longer function and your code needs maintenance. That's where Scrapling auto-matching feature comes into play.
+
+```python
+# Before the change
+page = Adaptor(page_source, url='example.com', auto_match=True)
+element = page.css('#p1' auto_save=True)
+if not element:  # One day website changes?
+    element = page.css('#p1', auto_match=True)  # Still finds it!
+# the rest of the code...
+```
+> How does the auto-matching work? Check the [FAQs](#FAQs) section for that and other possible issues while auto-matching.
+
+**Notes:**
+1. Passing the `auto_save` argument without setting `auto_match` to `True` while initializing the Adaptor object will only result in ignoring the `auto_save` argument value and the following warning message
+    ```text
+    Argument `auto_save` will be ignored because `auto_match` wasn't enabled on initialization. Check docs for more info.
+    ```
+    This behavior is purely for performance reasons so the database gets created/connected only when you are planning to use the auto-matching features. Same case with the `auto_match` argument.
+
+2. The `auto_match` parameter works only for `Adaptor` instances not `Adaptors` so if you do something like this you will get an error
+    ```python
+    page.css('body').css('#p1', auto_match=True)
+    ```
+    because you can't auto-match a whole list, you have to be specific and do something like
+    ```python
+    page.css('body')[0].css('#p1', auto_match=True)
+    ```
+
+### Is That All?
+Here's what else you can do with Scrapling:
+
+- Accessing the `lxml.etree` object itself of any element directly
+    ```python
+    >>> quote._root
+    <Element div at 0x107f98870>
+    ```
+- Saving and retrieving elements manually to auto-match them outside the `css` and the `xpath` methods but you have to set the identifier by yourself.
+
+  - To save element to the database:
+    ```python
+    >>> element = page.find_by_text('Tipping the Velvet', first_match=True)
+    >>> page.save(element, 'my_special_element')
+    ```
+  - Now later when you want to retrieve it and relocate it in the page with auto-matching, it would be like this
+    ```python
+    >>> element_dict = page.retrieve('my_special_element')
+    >>> page.relocate(element_dict, adaptor_type=True)
+    [<data='<a href="catalogue/tipping-the-velvet_99...' parent='<h3><a href="catalogue/tipping-the-velve...'>]
+    >>> page.relocate(element_dict, adaptor_type=True).css('::text')
+    ['Tipping the Velvet']
+    ```
+  - if you want to keep it as `lxml.etree` object, leave the `adaptor_type` argument
+    ```python
+    >>> page.relocate(element_dict)
+    [<Element a at 0x105a2a7b0>]
+    ```
+
+- Doing operations on element content is the same as scrapy
+    ```python
+    quote.re(r'somethings')  # Get all strings (TextHandlers) that match the regex pattern
+    quote.re_first(r'something')  # Get the first string (TextHandler) only
+    quote.json()  # If the content text is jsonable, then convert it to json using `orjson` which is 10x faster than the standard json library and provides more options
+    ```
+    Hence all of these methods are actually methods from the `TextHandler` within that contains the text content so the same can be done directly if you call the `.text` property or equivalent selector function.
+
+
+- Doing operations on the text content itself includes
+  - Cleaning the text from any white spaces and replacing consecutive spaces with single space
+    ```python
+    quote.clean()
+    ```
+  - You already know about the regex matching and the fast json parsing but did you know that all strings returned from the regex search are actually `TextHandler` objects too? so in cases where you have for example a JS object assigned to a JS variable inside JS code and want to extract it with regex and then convert it to json object, in other libraries, these would be more than 1 line of code but here you can do it in 1 line like this
+    ```python
+    page.xpath('//script/text()').re_first(r'var dataLayer = (.+);').json()
+    ```
+  - Sort all characters in the string as if it were a list and return the new string
+    ```python
+    quote.sort()
+    ```
+  > To be clear, `TextHandler` is a sub-class of Python's `str` so all normal operations/methods that work with Python strings will work with it.
+
+- Any element's attributes are not exactly a dictionary but a sub-class of [mapping](https://docs.python.org/3/glossary.html#term-mapping) called `AttributesHandler` that's read-only so it's faster and string values returned are actually `TextHandler` objects so all operations above can be done on them, standard dictionary operations that doesn't modify the data, and more :)
+  - Unlike standard dictionaries, here you can search by values too and can do partial searches. It might be handy in some cases (returns a generator of matches)
+    ```python
+    >>> for item in element.attrib.search_values('catalogue', partial=True):
+            print(item)
+    {'href': 'catalogue/tipping-the-velvet_999/index.html'}
+    ```
+  - Serialize the current attributes to JSON bytes:
+    ```python
+    >>> element.attrib.json_string
+    b'{"href":"catalogue/tipping-the-velvet_999/index.html","title":"Tipping the Velvet"}'
+    ```
+  - Converting it to a normal dictionary
+    ```python
+    >>> dict(element.attrib)
+    {'href': 'catalogue/tipping-the-velvet_999/index.html',
+    'title': 'Tipping the Velvet'}
+    ```
+
+Scrapling is under active development so expect many more features coming soon :)
+
+## More Advanced Usage
+
+There are a lot of deep details skipped here to make this as short as possible so to take a deep dive, head to the [docs](/docs) section. I will try to keep it updated as possible and add complex examples. There I will explain points like how to write your storage system, write spiders that don't depend on selectors at all, and more...
+
+Note that implementing your storage system can be complex as there are some strict rules such as inheriting from the same abstract class, following the singleton design pattern used in other classes, and more. So make sure to read the docs first.
+
+
+## FAQs
+This section addresses common questions about Scrapling, please read this section before opening an issue.
+
+### How does auto-matching work?
+  1. You need to get a working selector and run it at least once with methods `css` or `xpath` with the `auto_save` parameter set to `True` before structural changes happen.
+  2. Before returning results for you, Scrapling uses its configured database and saves unique properties about that element.
+  3. Now because everything about the element can be changed or removed, nothing from the element can be used as a unique identifier for the database. To solve this issue, I made the storage system rely on two things:
+     1. The domain of the URL you gave while initializing the first Adaptor object
+     2. The `identifier` parameter you passed to the method while selecting. If you didn't pass one, then the selector string itself will be used as an identifier but remember you will have to use it as an identifier value later when the structure changes and you want to pass the new selector.
+
+     Together both are used to retrieve the element's unique properties from the database later.
+  4. Now later when you enable the `auto_match` parameter for both the Adaptor instance and the method call. The element properties are retrieved and Scrapling loops over all elements in the page and compares each one's unique properties to the unique properties we already have for this element and a score is calculated for each one.
+  5. The comparison between elements is not exact but more about finding how similar these values are, so everything is taken into consideration even the values' order like the order in which the element class names were written before and the order in which the same element class names are written now.
+  6. The score for each element is stored in the table and in the end, the element(s) with the highest combined similarity scores are returned.
+
+### How does the auto-matching work if I didn't pass a URL while initializing the Adaptor object?
+Not a big problem as it depends on your usage. The word `default` will be used in place of the URL field while saving the element's unique properties. So this will only be an issue if you used the same identifier later for a different website that you didn't pass the URL parameter while initializing it as well. The save process will overwrite the previous data and auto-matching uses the latest saved properties only.
+
+### If all things about an element can change or get removed, what are the unique properties to be saved?
+For each element, Scrapling will extract:
+- Element tag name, text, attributes (names and values), siblings (tag names only), and path (tag names only).
+- Element's parent tag name, attributes (names and values), and text.
+
+### I have enabled the `auto_save`/`auto_match` parameter while selecting and it got completely ignored with a warning message
+That's because passing the `auto_save`/`auto_match` argument without setting `auto_match` to `True` while initializing the Adaptor object will only result in ignoring the `auto_save`/`auto_match` argument value. This behavior is purely for performance reasons so the database gets created only when you are planning to use the auto-matching features.
+
+### I have done everything as the docs but the auto-matching didn't return anything, what's wrong?
+It could be one of these reasons:
+1. No data were saved/stored for this element before.
+2. The selector passed is not the one used while storing element data. The solution is simple
+   - Pass the old selector again as an identifier to the method called.
+   - Retrieve the element with the retrieve method using the old selector as identifier then save it again with the save method and the new selector as identifier.
+   - Start using the identifier argument more often if you are planning to use every new selector from now on.
+3. The website had some extreme structural changes like a new full design. If this happens a lot with this website, the solution would be to make your code as selector-free as possible using Scrapling features.
+
+### Can Scrapling replace code built on top of BeautifulSoup4?
+Pretty much yeah, almost all features you get from BeautifulSoup can be found or achieved in Scrapling one way or another. In fact, if you see there's a feature in bs4 that is missing in Scrapling, please make a feature request from the issues tab to let me know.
+
+### Can Scrapling replace code built on top of AutoScraper?
+Of course, you can find elements by text/regex, find similar elements in a more reliable way than AutoScraper, and finally save/retrieve elements manually to use later as the model feature in AutoScraper. I have pulled all top articles about AutoScraper from Google and tested Scrapling against examples in them. In all examples, Scrapling got the same results as AutoScraper in much less time.
+
+### Is Scrapling thread-safe?
+Yes, Scrapling instances are thread-safe. Each Adaptor instance maintains its own state.
+
+## Contributing
+Everybody is invited and welcome to contribute to Scrapling. There is a lot to do!
+
+Please read the [contributing file](/CONTRIBUTING.md) before doing anything.
+
+## License
+This work is licensed under BSD-3
+
+## Acknowledgments
+This project includes code adapted from:
+- Parsel (BSD License) - Used for [translator](/scrapling/translator.py) submodule
+
+## Known Issues
+- In the auto-matching save process, the unique properties of the first element from the selection results are the only ones that get saved. So if the selector you are using selects different elements on the page that are in different locations, auto-matching will probably return to you the first element only when you relocate it later. This doesn't include combined CSS selectors (Using commas to combine more than one selector for example) as these selectors get separated and each selector gets executed alone.
+- Currently, Scrapling is not compatible with async/await.
+
+<div align="center"><small>Made with ❤️ by Karim Shoair</small></div><br>

ROADMAP.md

@@ -0,0 +1,13 @@
+## TODOs
+- Add more tests and increase the code coverage.
+- Structure the tests folder in a better way.
+- Add more documentation.
+- Add the browsing ability.
+- Create detailed documentation for 'readthedocs' website, preferably add Github action for deploying it.
+- Create a Scrapy plugin/decorator to make it replace parsel in the response argument when needed.
+- Need to add more functionality to `AttributesHandler` and more navigation functions to `Adaptor` object (ex: functions similar to map, filter, and reduce functions but here pass it to the element and the function is executed on children, siblings, next elements, etc...)
+- Add `.filter` method to `Adaptors` object and other similar methods.
+- Add functionality to automatically detect pagination URLs
+- Add the ability to auto-detect schemas in pages and manipulate them
+- Add ability to generate a regex from a group of elements (Like for all href attributes)
+- 

benchmarks.py

@@ -0,0 +1,139 @@
+import time
+import timeit
+import functools
+import requests
+from statistics import mean
+
+from scrapling import Adaptor
+from parsel import Selector
+from lxml import etree, html
+from bs4 import BeautifulSoup
+from pyquery import PyQuery as pq
+from autoscraper import AutoScraper
+from selectolax.parser import HTMLParser
+from mechanicalsoup import StatefulBrowser
+
+large_html = '<html><body>' + '<div class="item">' * 5000 + '</div>' * 5000 + '</body></html>'
+
+
+def benchmark(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        benchmark_name = func.__name__.replace('test_', '').replace('_', ' ')
+        print(f"-> {benchmark_name}", end=" ", flush=True)
+        # Warm-up phase
+        timeit.repeat(lambda: func(*args, **kwargs), number=2, repeat=2, globals=globals())
+        # Measure time (1 run, repeat 100 times, take average)
+        times = timeit.repeat(
+            lambda: func(*args, **kwargs), number=1, repeat=100, globals=globals(), timer=time.process_time
+        )
+        min_time = round(mean(times) * 1000, 2)  # Convert to milliseconds
+        print(f"average execution time: {min_time} ms")
+        return min_time
+
+    return wrapper
+
+
+@benchmark
+def test_lxml():
+    return [
+        e.text
+        for e in etree.fromstring(
+            large_html,
+            # Scrapling and Parsel use the same parser inside so this is just to make it fair
+            parser=html.HTMLParser(recover=True, huge_tree=True)
+        ).cssselect('.item')]
+
+
+@benchmark
+def test_bs4_lxml():
+    return [e.text for e in BeautifulSoup(large_html, 'lxml').select('.item')]
+
+
+@benchmark
+def test_bs4_html5lib():
+    return [e.text for e in BeautifulSoup(large_html, 'html5lib').select('.item')]
+
+
+@benchmark
+def test_pyquery():
+    return [e.text() for e in pq(large_html)('.item').items()]
+
+
+@benchmark
+def test_scrapling():
+    # No need to do `.extract()` like parsel to extract text
+    # Also, this is faster than `[t.text for t in Adaptor(large_html, auto_match=False, debug=False).css('.item')]`
+    # for obvious reasons, of course.
+    return Adaptor(large_html, auto_match=False, debug=False).css('.item::text')
+
+
+@benchmark
+def test_parsel():
+    return Selector(text=large_html).css('.item::text').extract()
+
+
+@benchmark
+def test_mechanicalsoup():
+    browser = StatefulBrowser()
+    browser.open_fake_page(large_html)
+    return [e.text for e in browser.page.select('.item')]
+
+
+@benchmark
+def test_selectolax():
+    return [node.text() for node in HTMLParser(large_html).css('.item')]
+
+
+def display(results):
+    # Sort and display results
+    sorted_results = sorted(results.items(), key=lambda x: x[1])  # Sort by time
+    scrapling_time = results['Scrapling']
+    print("\nRanked Results (fastest to slowest):")
+    print(f" i. {'Library tested':<18} | {'avg. time (ms)':<15} | vs Scrapling")
+    print('-' * 50)
+    for i, (test_name, test_time) in enumerate(sorted_results, 1):
+        compare = round(test_time / scrapling_time, 3)
+        print(f" {i}. {test_name:<18} | {str(test_time):<15} | {compare}")
+
+
+@benchmark
+def test_scrapling_text(request_html):
+    # Will loop over resulted elements to get text too to make comparison even more fair otherwise Scrapling will be even faster
+    return [
+        element.text for element in Adaptor(
+            request_html, auto_match=False, debug=False
+        ).find_by_text('Tipping the Velvet', first_match=True).find_similar(ignore_attributes=['title'])
+    ]
+
+
+@benchmark
+def test_autoscraper(request_html):
+    # autoscraper by default returns elements text
+    return AutoScraper().build(html=request_html, wanted_list=['Tipping the Velvet'])
+
+
+if __name__ == "__main__":
+    print(' Benchmark: Speed of parsing and retrieving the text content of 5000 nested elements \n')
+    results1 = {
+        "Raw Lxml": test_lxml(),
+        "Parsel/Scrapy": test_parsel(),
+        "Scrapling": test_scrapling(),
+        'Selectolax': test_selectolax(),
+        "PyQuery": test_pyquery(),
+        "BS4 with Lxml": test_bs4_lxml(),
+        "MechanicalSoup": test_mechanicalsoup(),
+        "BS4 with html5lib": test_bs4_html5lib(),
+    }
+
+    display(results1)
+    print('\n' + "="*25)
+    req = requests.get('https://books.toscrape.com/index.html')
+    print(
+        ' Benchmark: Speed of searching for an element by text content, and retrieving the text of similar elements\n'
+    )
+    results2 = {
+        "Scrapling": test_scrapling_text(req.text),
+        "AutoScraper": test_autoscraper(req.text),
+    }
+    display(results2)

docs/Core/using scrapling custom types.md

@@ -0,0 +1,21 @@
+> You can take advantage from the custom-made types for Scrapling and use it outside the library if you want. It's better than copying their code after all :)
+
+### All current types can be imported alone like below
+```python
+>>> from scrapling import TextHandler, AttributesHandler
+
+>>> somestring = TextHandler('{}')
+>>> somestring.json()
+'{}'
+>>> somedict_1 = AttributesHandler({'a': 1})
+>>> somedict_2 = AttributesHandler(a=1)
+```
+
+Note `TextHandler` is a sub-class of Python's `str` so all normal operations/methods that work with Python strings will work.
+If you want to check for the type in your code, it's better to depend on Python built-in function `issubclass`.
+
+The class `AttributesHandler` is a sub-class of `collections.abc.Mapping` so it's immutable (read-only) and all operations are inherited from it. The data passed can be accessed later though the `._data` method but careful it's of type `types.MappingProxyType` so it's immutable (read-only) as well (faster than `collections.abc.Mapping` by fractions of seconds).
+
+So basically to make it simple to you if you are new to Python, the same operations and methods from Python standard `dict` type will all work with class `AttributesHandler` except the ones that try to modify the actual data.
+
+If you want to modify the data inside `AttributesHandler`, you have to convert it to dictionary first like with using the `dict` function and modify it outside.

docs/Examples/selectorless_stackoverflow.py

@@ -0,0 +1,23 @@
+"""
+I only made this example to show how Scrapling features can be used to scrape a website without writing any selector
+    so this script doesn't depend on the website structure.
+"""
+
+import requests
+from scrapling import Adaptor
+
+response = requests.get('https://stackoverflow.com/questions/tagged/web-scraping?sort=MostVotes&filters=NoAcceptedAnswer&edited=true&pagesize=50&page=2')
+page = Adaptor(response.text, url=response.url)
+# First we will extract the first question title and its author based on the text content
+first_question_title = page.find_by_text('Run Selenium Python Script on Remote Server')
+first_question_author = page.find_by_text('Ryan')
+# If you want you can extract other questions tags like below
+first_question = first_question_title.find_ancestor(
+    lambda ancestor: ancestor.attrib.get('id') and 'question-summary' in ancestor.attrib.get('id')
+)
+rest_of_questions = first_question.find_similar()
+# But since nothing to rely on to extract other titles/authors from these elements without CSS/XPath selectors due to the website nature
+# We will get all the rest of the titles/authors in the page depending on the first title and the first author we got above as a starting point
+for i, (title, author) in enumerate(zip(first_question_title.find_similar(), first_question_author.find_similar()), start=1):
+    print(i, title.text, author.text)
+

docs/Extending Scrapling/writing storage system.md

@@ -0,0 +1,17 @@
+Scrapling by default is using SQLite but in case you want to write your storage system to store elements properties there for the auto-matching, this tutorial got you covered.
+
+You might want to use FireBase for example and share the database between multiple spiders on different machines, it's a great idea to use an online database like that because this way the spiders will share with each others.
+
+So first to make your storage class work, it must do the big 3:
+1. Inherit from the abstract class `scrapling.storage_adaptors.StorageSystemMixin` and accept a string argument which will be the `url` argument to maintain the library logic.
+2. Use the decorator `functools.lru_cache` on top of the class itself to follow the Singleton design pattern as other classes.
+3. Implement methods `save` and `retrieve`, as you see from the type hints:
+   - The method `save` returns nothing and will get two arguments from the library
+        * The first one is of type `lxml.html.HtmlElement` which is the element itself, ofc. It must be converted to dictionary using the function `scrapling.utils._StorageTools.element_to_dict` so we keep the same format then saved to your database as you wish.
+        * The second one is string which is the identifier used for retrieval. The combination of this identifier and the `url` argument from initialization must be unique for each row or the auto-match will be messed up.
+   - The method `retrieve` takes a string which is the identifier, using it with the `url` passed on initialization the element's dictionary is retrieved from the database and returned if it exist otherwise it returns `None`
+> If the instructions weren't clear enough for you, you can check my implementation using SQLite3 in [storage_adaptors](/scrapling/storage_adaptors.py) file
+
+If your class satisfy this, the rest is easy. If you are planning to use the library in a threaded application, make sure that your class supports it. The default used class is thread-safe.
+
+There are some helper functions added to the abstract class if you want to use it. It's easier to see it for yourself in the [code](/scrapling/storage_adaptors.py), it's heavily commented :)

docs/index.md

@@ -0,0 +1,2 @@
+# This section is still under work but any help is highly appreciated
+## I will try to make full detailed documentation with Sphinx ASAP.

pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+addopts = -p no:warnings --doctest-modules --ignore=setup.py

scrapling/__init__.py

@@ -0,0 +1,10 @@
+# Declare top-level shortcuts
+from scrapling.parser import Adaptor, Adaptors
+from scrapling.custom_types import TextHandler, AttributesHandler
+
+__author__ = "Karim Shoair (karim.shoair@pm.me)"
+__version__ = "0.1"
+__copyright__ = "Copyright (c) 2024 Karim Shoair"
+
+
+__all__ = ['Adaptor', 'Adaptors', 'TextHandler', 'AttributesHandler']

scrapling/custom_types.py

@@ -0,0 +1,146 @@
+import re
+from types import MappingProxyType
+from collections.abc import Mapping
+from typing import Dict, List, Union, Pattern
+
+from scrapling.utils import _is_iterable, flatten
+
+from orjson import loads, dumps
+from w3lib.html import replace_entities as _replace_entities
+
+
+class TextHandler(str):
+    """Extends standard Python string by adding more functionality"""
+    __slots__ = ()
+
+    def __new__(cls, string):
+        # Because str is immutable and we can't override __init__
+        if type(string) is str:
+            return super().__new__(cls, string)
+        else:
+            return super().__new__(cls, '')
+
+    def sort(self, reverse: bool = False) -> str:
+        """Return a sorted version of the string"""
+        return self.__class__("".join(sorted(self, reverse=reverse)))
+
+    def clean(self) -> str:
+        """Return a new version of the string after removing all white spaces and consecutive spaces"""
+        data = re.sub(r'[\t|\r|\n]', '', self)
+        data = re.sub(' +', ' ', data)
+        return self.__class__(data.strip())
+
+    def json(self) -> Dict:
+        """Return json response if the response is jsonable otherwise throw error"""
+        # Using __str__ function as a workaround for orjson issue with subclasses of str
+        # Check this out: https://github.com/ijl/orjson/issues/445
+        return loads(self.__str__())
+
+    def re(
+            self, regex: Union[str, Pattern[str]], replace_entities: bool = True, clean_match: bool = False,
+            case_sensitive: bool = False, check_match: bool = False
+    ) -> Union[List[str], bool]:
+        """Apply the given regex to the current text and return a list of strings with the matches.
+
+        :param regex: Can be either a compiled regular expression or a string.
+        :param replace_entities: if enabled character entity references are replaced by their corresponding character
+        :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
+        :param case_sensitive: if enabled, function will set the regex to ignore letters case while compiling it
+        :param check_match: used to quickly check if this regex matches or not without any operations on the results
+
+        """
+        if isinstance(regex, str):
+            if not case_sensitive:
+                regex = re.compile(regex, re.UNICODE)
+            else:
+                regex = re.compile(regex, flags=re.UNICODE | re.IGNORECASE)
+
+        input_text = self.clean() if clean_match else self
+        results = regex.findall(input_text)
+        if check_match:
+            return bool(results)
+
+        if all(_is_iterable(res) for res in results):
+            results = flatten(results)
+
+        if not replace_entities:
+            return [TextHandler(string) for string in results]
+
+        return [TextHandler(_replace_entities(s)) for s in results]
+
+    def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True,
+                 clean_match: bool = False, case_sensitive: bool = False,):
+        """Apply the given regex to text and return the first match if found, otherwise return the default value.
+
+        :param regex: Can be either a compiled regular expression or a string.
+        :param default: The default value to be returned if there is no match
+        :param replace_entities: if enabled character entity references are replaced by their corresponding character
+        :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
+        :param case_sensitive: if enabled, function will set the regex to ignore letters case while compiling it
+
+        """
+        result = self.re(regex, replace_entities, clean_match=clean_match, case_sensitive=case_sensitive)
+        return result[0] if result else default
+
+
+class AttributesHandler(Mapping):
+    """A read-only mapping to use instead of the standard dictionary for the speed boost but
+     at the same time I use it to add more functionalities.
+    If standard dictionary is needed, just convert this class to dictionary with `dict` function
+    """
+    __slots__ = ('_data',)
+
+    def __init__(self, mapping=None, **kwargs):
+        mapping = {
+            key: TextHandler(value) if type(value) is str else value
+            for key, value in mapping.items()
+        } if mapping is not None else {}
+
+        if kwargs:
+            mapping.update({
+                key: TextHandler(value) if type(value) is str else value
+                for key, value in kwargs.items()
+            })
+
+        # Fastest read-only mapping type
+        self._data = MappingProxyType(mapping)
+
+    def get(self, key, default=None):
+        """Acts like standard dictionary `.get()` method"""
+        return self._data.get(key, default)
+
+    def search_values(self, keyword, partial=False):
+        """Search current attributes by values and return dictionary of each matching item
+        :param keyword: The keyword to search for in the attributes values
+        :param partial: If True, the function will search if keyword in each value instead of perfect match
+        """
+        for key, value in self._data.items():
+            if partial:
+                if keyword in value:
+                    yield AttributesHandler({key: value})
+            else:
+                if keyword == value:
+                    yield AttributesHandler({key: value})
+
+    @property
+    def json_string(self):
+        """Convert current attributes to JSON string if the attributes are JSON serializable otherwise throws error"""
+        return dumps(dict(self._data))
+
+    def __getitem__(self, key):
+        return self._data[key]
+
+    def __iter__(self):
+        return iter(self._data)
+
+    def __len__(self):
+        return len(self._data)
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}({self._data})"
+
+    def __str__(self):
+        return str(self._data)
+
+    def __contains__(self, key):
+        return key in self._data

scrapling/mixins.py

@@ -0,0 +1,74 @@
+
+class SelectorsGeneration:
+    """Selectors generation functions
+    Trying to generate selectors like Firefox or maybe cleaner ones!? Ehm
+    Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591"""
+
+    def __general_selection(self, selection: str = 'css') -> str:
+        """Generate a selector for the current element.
+        :return: A string of the generated selector.
+        """
+        selectorPath = []
+        target = self
+        css = selection.lower() == 'css'
+        while target is not None:
+            if target.parent:
+                if target.attrib.get('id'):
+                    # id is enough
+                    part = (
+                        f'#{target.attrib["id"]}' if css
+                        else f"[@id='{target.attrib['id']}']"
+                    )
+                    selectorPath.append(part)
+                    return (
+                        " > ".join(reversed(selectorPath)) if css
+                        else '//*' + "/".join(reversed(selectorPath))
+                    )
+                else:
+                    part = f'{target.tag}'
+                    # We won't use classes anymore because I some websites share exact classes between elements
+                    # classes = target.attrib.get('class', '').split()
+                    # if classes and css:
+                    #     part += f".{'.'.join(classes)}"
+                    # else:
+                    counter = {}
+                    for child in target.parent.children:
+                        counter.setdefault(child.tag, 0)
+                        counter[child.tag] += 1
+                        if child._root == target._root:
+                            break
+
+                    if counter[target.tag] > 1:
+                        part += (
+                            f":nth-of-type({counter[target.tag]})" if css
+                            else f"[{counter[target.tag]}]"
+                        )
+
+                selectorPath.append(part)
+                target = target.parent
+                if target is None or target.tag == 'html':
+                    return (
+                        " > ".join(reversed(selectorPath)) if css
+                        else '//' + "/".join(reversed(selectorPath))
+                    )
+            else:
+                break
+
+        return (
+            " > ".join(reversed(selectorPath)) if css
+            else '//' + "/".join(reversed(selectorPath))
+        )
+
+    @property
+    def css_selector(self) -> str:
+        """Generate a CSS selector for the current element
+        :return: A string of the generated selector.
+        """
+        return self.__general_selection()
+
+    @property
+    def xpath_selector(self) -> str:
+        """Generate a XPath selector for the current element
+        :return: A string of the generated selector.
+        """
+        return self.__general_selection('xpath')

scrapling/parser.py

@@ -0,0 +1,903 @@
+import os
+from difflib import SequenceMatcher
+from typing import Any, Dict, List, Tuple, Optional, Pattern, SupportsIndex, Union, Callable, Generator
+
+from scrapling.translator import HTMLTranslator
+from scrapling.mixins import SelectorsGeneration
+from scrapling.custom_types import TextHandler, AttributesHandler
+from scrapling.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools
+from scrapling.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden
+
+from lxml import etree, html
+from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors
+
+
+class Adaptor(SelectorsGeneration):
+    __slots__ = (
+        'url', 'encoding', '__auto_match_enabled', '_root', '_storage', '__debug',
+        '__keep_comments', '__huge_tree_enabled', '__attributes', '__text', '__tag',
+    )
+
+    def __init__(
+            self,
+            text: Optional[str] = None,
+            url: Optional[str] = None,
+            body: bytes = b"",
+            encoding: str = "utf8",
+            huge_tree: bool = True,
+            root: Optional[html.HtmlElement] = None,
+            keep_comments: Optional[bool] = False,
+            auto_match: Optional[bool] = False,
+            storage: Any = SQLiteStorageSystem,
+            storage_args: Optional[Dict] = None,
+            debug: Optional[bool] = True,
+    ):
+        """The main class that works as a wrapper for the HTML input data. Using this class, you can search for elements
+        with expressions in CSS, XPath, or with simply text. Check the docs for more info.
+
+        Here we try to extend module ``lxml.html.HtmlElement`` while maintaining a simpler interface, We are not
+        inheriting from the ``lxml.html.HtmlElement`` because it's not pickleable which makes a lot of reference jobs
+        not possible. You can test it here and see code explodes with `AssertionError: invalid Element proxy at...`.
+        It's an old issue with lxml, see `this entry <https://bugs.launchpad.net/lxml/+bug/736708>`
+
+        :param text: HTML body passed as text.
+        :param url: allows storing a URL with the html data for retrieving later.
+        :param body: HTML body as ``bytes`` object. It can be used instead of the ``text`` argument.
+        :param encoding: The encoding type that will be used in HTML parsing, default is `UTF-8`
+        :param huge_tree: Enabled by default, should always be enabled when parsing large HTML documents. This controls
+            libxml2 feature that forbids parsing certain large documents to protect from possible memory exhaustion.
+        :param root: Used internally to pass etree objects instead of text/body arguments, it takes highest priority.
+            Don't use it unless you know what you are doing!
+        :param keep_comments: While parsing the HTML body, drop comments or not. Disabled by default for obvious reasons
+        :param auto_match: Globally turn-off the auto-match feature in all functions, this argument takes higher
+            priority over all auto-match related arguments/functions in the class.
+        :param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info.
+        :param storage_args: A dictionary of ``argument->value`` pairs to be passed for the storage class.
+            If empty, default values will be used.
+        :param debug: Enable debug mode
+        """
+        if root is None and not body and text is None:
+            raise ValueError("Adaptor class needs text, body, or root arguments to work")
+
+        if root is None:
+            if text is None:
+                if not body or not isinstance(body, bytes):
+                    raise TypeError(f"body argument must be valid and of type bytes, got {body.__class__}")
+
+                body = body.replace(b"\x00", b"").strip()
+            else:
+                if not isinstance(text, str):
+                    raise TypeError(f"text argument must be of type str, got {text.__class__}")
+
+                body = text.strip().replace("\x00", "").encode(encoding) or b"<html/>"
+
+            parser = html.HTMLParser(
+                # https://lxml.de/api/lxml.etree.HTMLParser-class.html
+                recover=True, remove_blank_text=True, remove_comments=(keep_comments is True), encoding=encoding,
+                compact=True, huge_tree=huge_tree, default_doctype=True
+            )
+            self._root = etree.fromstring(body, parser=parser, base_url=url)
+
+        else:
+            # All html types inherits from HtmlMixin so this to check for all at once
+            if not issubclass(type(root), html.HtmlMixin):
+                raise TypeError(
+                    f"Root have to be a valid element of `html` module types to work, not of type {type(root)}"
+                )
+
+            self._root = root
+
+        setup_basic_logging(level='debug' if debug else 'info')
+        self.__auto_match_enabled = auto_match
+
+        if self.__auto_match_enabled:
+            if not storage_args:
+                storage_args = {
+                    'storage_file': os.path.join(os.path.dirname(__file__), 'elements_storage.db'),
+                    'url': url
+                }
+
+            if not hasattr(storage, '__wrapped__'):
+                raise ValueError("Storage class must be wrapped with cache decorator, see docs for info")
+
+            if not issubclass(storage.__wrapped__, StorageSystemMixin):
+                raise ValueError("Storage system must be inherited from class `StorageSystemMixin`")
+
+            self._storage = storage(**storage_args)
+
+        self.__keep_comments = keep_comments
+        self.__huge_tree_enabled = huge_tree
+        self.encoding = encoding
+        self.url = url
+        # For selector stuff
+        self.__attributes = None
+        self.__text = None
+        self.__tag = None
+        self.__debug = debug
+
+    # Node functionalities, I wanted to move to separate Mixin class but it had slight impact on performance
+    @staticmethod
+    def _is_text_node(element: Union[html.HtmlElement, etree._ElementUnicodeResult]) -> bool:
+        """Return True if given element is a result of a string expression
+        Examples:
+            Xpath -> '/text()', '/@attribute' etc...
+            CSS3  -> '::text', '::attr(attrib)'...
+        """
+        # Faster than checking `element.is_attribute or element.is_text or element.is_tail`
+        return issubclass(type(element), etree._ElementUnicodeResult)
+
+    def __get_correct_result(
+            self, element: Union[html.HtmlElement, etree._ElementUnicodeResult]
+    ) -> Union[TextHandler, html.HtmlElement, 'Adaptor', str]:
+        """Used internally in all functions to convert results to type (Adaptor|Adaptors) when possible"""
+        if self._is_text_node(element):
+            # etree._ElementUnicodeResult basically inherit from `str` so it's fine
+            return TextHandler(str(element))
+        else:
+            if issubclass(type(element), html.HtmlMixin):
+                return self.__class__(
+                    root=element, url=self.url, encoding=self.encoding, auto_match=self.__auto_match_enabled,
+                    keep_comments=self.__keep_comments, huge_tree=self.__huge_tree_enabled, debug=self.__debug
+                )
+            return element
+
+    def __convert_results(
+            self, result: Union[List[html.HtmlElement], html.HtmlElement]
+    ) -> Union['Adaptors[Adaptor]', 'Adaptor', List, None]:
+        """Used internally in all functions to convert results to type (Adaptor|Adaptors) in bulk when possible"""
+        if result is None:
+            return None
+        elif result == []:  # Lxml will give a warning if I used something like `not result`
+            return []
+
+        if isinstance(result, Adaptors):
+            return result
+
+        if type(result) is list:
+            results = [self.__get_correct_result(n) for n in result]
+            if all(isinstance(res, self.__class__) for res in results):
+                return Adaptors(results)
+            return results
+
+        return self.__get_correct_result(result)
+
+    def __getstate__(self) -> Any:
+        # lxml don't like it :)
+        raise TypeError("Can't pickle Adaptor objects")
+
+    # The following four properties I made them into functions instead of variables directly
+    # So they don't slow down the process of initializing many instances of the class and gets executed only
+    # when the user need them for the first time for that specific element and gets cached for next times
+    # Doing that only made the library performance test sky rocked multiple times faster than before
+    # because I was executing them on initialization before :))
+    @property
+    def tag(self) -> str:
+        """Get tag name of the element"""
+        if not self.__tag:
+            self.__tag = self._root.tag
+        return self.__tag
+
+    @property
+    def text(self) -> TextHandler:
+        """Get text content of the element"""
+        if not self.__text:
+            self.__text = TextHandler(self._root.text)
+        return self.__text
+
+    def get_all_text(self, separator: str = "\n", strip: bool = False, ignore_tags: Tuple = ('script', 'style',), valid_values: bool = True) -> TextHandler:
+        """Get all child strings of this element, concatenated using the given separator.
+
+        :param separator: Strings will be concatenated using this separator.
+        :param strip: If True, strings will be stripped before being concatenated.
+        :param ignore_tags: A tuple of all tag names you want to ignore
+        :param valid_values: If enabled, elements with text-content that is empty or only whitespaces will be ignored
+
+        :return: A TextHandler
+        """
+        _all_strings = []
+
+        def _traverse(node: html.HtmlElement) -> None:
+            """Traverse element children and get text content of each
+
+            :param node: Current node in the tree structure
+            :return:
+            """
+            if node.tag not in ignore_tags:
+                text = node.text
+                if text and type(text) is str:
+                    if valid_values:
+                        if text.strip():
+                            _all_strings.append(text if not strip else text.strip())
+                    else:
+                        _all_strings.append(text if not strip else text.strip())
+
+            for branch in node.iterchildren():
+                _traverse(branch)
+
+        # We will start using Lxml directly for the speed boost
+        _traverse(self._root)
+
+        return TextHandler(separator.join([s for s in _all_strings]))
+
+    @property
+    def attrib(self) -> AttributesHandler:
+        """Get attributes of the element"""
+        if not self.__attributes:
+            self.__attributes = AttributesHandler(self._root.attrib)
+        return self.__attributes
+
+    @property
+    def html_content(self) -> str:
+        """Return the inner html code of the element"""
+        return etree.tostring(self._root, encoding='unicode', method='html', with_tail=False)
+
+    body = html_content
+
+    def prettify(self) -> str:
+        """Return a prettified version of the element's inner html-code"""
+        return etree.tostring(self._root, encoding='unicode', pretty_print=True, method='html', with_tail=False)
+
+    def has_class(self, class_name: str) -> bool:
+        """Check if element has a specific class
+        :param class_name: The class name to check for
+        :return: True if element has class with that name otherwise False
+        """
+        return class_name in self._root.classes
+
+    @property
+    def parent(self) -> Union['Adaptor', None]:
+        """Return the direct parent of the element or ``None`` otherwise"""
+        return self.__convert_results(self._root.getparent())
+
+    @property
+    def children(self) -> Union['Adaptors[Adaptor]', List]:
+        """Return the children elements of the current element or empty list otherwise"""
+        return self.__convert_results(list(
+            child for child in self._root.iterchildren() if type(child) not in html_forbidden
+        ))
+
+    @property
+    def siblings(self) -> Union['Adaptors[Adaptor]', List]:
+        """Return other children of the current element's parent or empty list otherwise"""
+        if self.parent:
+            return Adaptors([child for child in self.parent.children if child._root != self._root])
+        return []
+
+    def iterancestors(self) -> Generator['Adaptor', None, None]:
+        """Return a generator that loops over all ancestors of the element, starting with element's parent."""
+        for ancestor in self._root.iterancestors():
+            yield self.__convert_results(ancestor)
+
+    def find_ancestor(self, func: Callable[['Adaptor'], bool]) -> Union['Adaptor', None]:
+        """Loop over all ancestors of the element till one match the passed function
+        :param func: A function that takes each ancestor as an argument and returns True/False
+        :return: The first ancestor that match the function or ``None`` otherwise.
+        """
+        for ancestor in self.iterancestors():
+            if func(ancestor):
+                return ancestor
+        return None
+
+    @property
+    def path(self) -> 'Adaptors[Adaptor]':
+        """Returns list of type :class:`Adaptors` that contains the path leading to the current element from the root."""
+        lst = list(self.iterancestors())
+        return Adaptors(lst)
+
+    @property
+    def next(self) -> Union['Adaptor', None]:
+        """Returns the next element of the current element in the children of the parent or ``None`` otherwise."""
+        next_element = self._root.getnext()
+        if next_element is not None:
+            while type(next_element) in html_forbidden:
+                # Ignore html comments and unwanted types
+                next_element = next_element.getnext()
+
+        return self.__convert_results(next_element)
+
+    @property
+    def previous(self) -> Union['Adaptor', None]:
+        """Returns the previous element of the current element in the children of the parent or ``None`` otherwise."""
+        prev_element = self._root.getprevious()
+        if prev_element is not None:
+            while type(prev_element) in html_forbidden:
+                # Ignore html comments and unwanted types
+                prev_element = prev_element.getprevious()
+
+        return self.__convert_results(prev_element)
+
+    def __str__(self) -> str:
+        return self.html_content
+
+    def __repr__(self) -> str:
+        length_limit = 40
+        data = "<"
+        content = clean_spaces(self.html_content)
+        if len(content) > length_limit:
+            content = content[:length_limit].strip() + '...'
+        data += f"data='{content}'"
+
+        if self.parent:
+            parent_content = clean_spaces(self.parent.html_content)
+            if len(parent_content) > length_limit:
+                parent_content = parent_content[:length_limit].strip() + '...'
+
+            data += f" parent='{parent_content}'"
+
+        return data + ">"
+
+    # From here we start the selecting functions
+    def relocate(
+            self, element: Union[Dict, html.HtmlElement, 'Adaptor'], percentage: int = 0, adaptor_type: bool = False
+    ) -> Union[List[Union[html.HtmlElement, None]], 'Adaptors']:
+        """This function will search again for the element in the page tree, used automatically on page structure change
+
+        :param element: The element we want to relocate in the tree
+        :param percentage: The minimum percentage to accept and not going lower than that. Be aware that the percentage
+         calculation depends solely on the page structure so don't play with this number unless you must know
+         what you are doing!
+        :param adaptor_type: If True, the return result will be converted to `Adaptors` object
+        :return: List of pure HTML elements that got the highest matching score or 'Adaptors' object
+        """
+        score_table = {}
+        # Note: `element` will be most likely always be a dictionary at this point.
+        if isinstance(element, self.__class__):
+            element = element._root
+
+        if issubclass(type(element), html.HtmlElement):
+            element = _StorageTools.element_to_dict(element)
+
+        # TODO: Optimize the traverse logic a bit, maybe later
+        def _traverse(node: html.HtmlElement, ele: Dict) -> None:
+            """Get the matching score of the given element against the node then traverse the children
+
+            :param node: Current node in the tree structure
+            :param ele: The element we are searching for as dictionary
+            :return:
+            """
+            # Hence: the code doesn't stop even if the score was 100%
+            # because there might be another element(s) left in page with the same score
+            score = self.__calculate_similarity_score(ele, node)
+            score_table.setdefault(score, []).append(node)
+            for branch in node.iterchildren():
+                _traverse(branch, ele)
+
+        # This will block until we traverse all children/branches
+        _traverse(self._root, element)
+
+        if score_table:
+            highest_probability = max(score_table.keys())
+            if score_table[highest_probability] and highest_probability >= percentage:
+                logging.debug(f'Highest probability was {highest_probability}%')
+                logging.debug('Top 5 best matching elements are: ')
+                for percent in tuple(sorted(score_table.keys(), reverse=True))[:5]:
+                    logging.debug(f'{percent} -> {self.__convert_results(score_table[percent])}')
+                if not adaptor_type:
+                    return score_table[highest_probability]
+                return self.__convert_results(score_table[highest_probability])
+        return []
+
+    def css(self, selector: str, identifier: str = '',
+            auto_match: bool = False, auto_save: bool = False, percentage: int = 0
+            ) -> Union['Adaptors[Adaptor]', List]:
+        """Search current tree with CSS3 selectors
+
+        **Important:
+        It's recommended to use the identifier argument if you plan to use different selector later
+        and want to relocate the same element(s)**
+
+        :param selector: The CSS3 selector to be used.
+        :param auto_match: Enabled will make function try to relocate the element if it was 'saved' before
+        :param identifier: A string that will be used to save/retrieve element's data in auto-matching
+         otherwise the selector will be used.
+        :param auto_save: Automatically save new elements for `auto_match` later
+        :param percentage: The minimum percentage to accept while auto-matching and not going lower than that.
+         Be aware that the percentage calculation depends solely on the page structure so don't play with this
+         number unless you must know what you are doing!
+
+        :return: List as :class:`Adaptors`
+        """
+        try:
+            if not self.__auto_match_enabled:
+                # No need to split selectors in this case, let's save some CPU cycles :)
+                xpath_selector = HTMLTranslator().css_to_xpath(selector)
+                return self.xpath(xpath_selector, identifier or selector, auto_match, auto_save, percentage)
+
+            results = []
+            if ',' in selector:
+                for single_selector in split_selectors(selector):
+                    # I'm doing this only so the `save` function save data correctly for combined selectors
+                    # Like using the ',' to combine two different selectors that point to different elements.
+                    xpath_selector = HTMLTranslator().css_to_xpath(single_selector.canonical())
+                    results += self.xpath(
+                        xpath_selector, identifier or single_selector.canonical(), auto_match, auto_save, percentage
+                    )
+            else:
+                xpath_selector = HTMLTranslator().css_to_xpath(selector)
+                return self.xpath(xpath_selector, identifier or selector, auto_match, auto_save, percentage)
+
+            return self.__convert_results(results)
+        except (SelectorError, SelectorSyntaxError,):
+            raise SelectorSyntaxError(f"Invalid CSS selector: {selector}")
+
+    def xpath(self, selector: str, identifier: str = '',
+              auto_match: bool = False, auto_save: bool = False, percentage: int = 0, **kwargs: Any
+              ) -> Union['Adaptors[Adaptor]', List]:
+        """Search current tree with XPath selectors
+
+        **Important:
+        It's recommended to use the identifier argument if you plan to use different selector later
+        and want to relocate the same element(s)**
+
+         Note: **Additional keyword arguments will be passed as XPath variables in the XPath expression!**
+
+        :param selector: The XPath selector to be used.
+        :param auto_match: Enabled will make function try to relocate the element if it was 'saved' before
+        :param identifier: A string that will be used to save/retrieve element's data in auto-matching
+         otherwise the selector will be used.
+        :param auto_save: Automatically save new elements for `auto_match` later
+        :param percentage: The minimum percentage to accept while auto-matching and not going lower than that.
+         Be aware that the percentage calculation depends solely on the page structure so don't play with this
+         number unless you must know what you are doing!
+
+        :return: List as :class:`Adaptors`
+        """
+        try:
+            selected_elements = self._root.xpath(selector, **kwargs)
+
+            if selected_elements:
+                if not self.__auto_match_enabled and auto_save:
+                    logging.warning("Argument `auto_save` will be ignored because `auto_match` wasn't enabled on initialization. Check docs for more info.")
+
+                elif self.__auto_match_enabled and auto_save:
+                    self.save(selected_elements[0], identifier or selector)
+
+                return self.__convert_results(selected_elements)
+            else:
+                if self.__auto_match_enabled and auto_match:
+                    element_data = self.retrieve(identifier or selector)
+                    if element_data:
+                        relocated = self.relocate(element_data, percentage)
+                        if relocated is not None and auto_save:
+                            self.save(relocated[0], identifier or selector)
+
+                        return self.__convert_results(relocated)
+                    else:
+                        return self.__convert_results(selected_elements)
+
+                elif not self.__auto_match_enabled and auto_match:
+                    logging.warning("Argument `auto_match` will be ignored because `auto_match` wasn't enabled on initialization. Check docs for more info.")
+
+                return self.__convert_results(selected_elements)
+
+        except (SelectorError, SelectorSyntaxError, etree.XPathError, etree.XPathEvalError):
+            raise SelectorSyntaxError(f"Invalid XPath selector: {selector}")
+
+    def __calculate_similarity_score(self, original: Dict, candidate: html.HtmlElement) -> float:
+        """Used internally to calculate a score that shows how candidate element similar to the original one
+
+        :param original: The original element in the form of the dictionary generated from `element_to_dict` function
+        :param candidate: The element to compare with the original element.
+        :return: A percentage score of how similar is the candidate to the original element
+        """
+        score, checks = 0, 0
+        candidate = _StorageTools.element_to_dict(candidate)
+
+        # Possible TODO:
+        # Study the idea of giving weight to each test below so some are more important than others
+        # Current results: With weights some websites had better score while it was worse for others
+        score += 1 if original['tag'] == candidate['tag'] else 0  # * 0.3  # 30%
+        checks += 1
+
+        if original['text']:
+            score += SequenceMatcher(None, original['text'], candidate.get('text') or '').ratio()  # * 0.3  # 30%
+            checks += 1
+
+        # if both doesn't have attributes, it still count for something!
+        score += self.__calculate_dict_diff(original['attributes'], candidate['attributes'])  # * 0.3  # 30%
+        checks += 1
+
+        # Separate similarity test for class, id, href,... this will help in full structural changes
+        for attrib in ('class', 'id', 'href', 'src',):
+            if original['attributes'].get(attrib):
+                score += SequenceMatcher(
+                    None, original['attributes'][attrib], candidate['attributes'].get(attrib) or ''
+                ).ratio()  # * 0.3  # 30%
+                checks += 1
+
+        score += SequenceMatcher(None, original['path'], candidate['path']).ratio()  # * 0.1  # 10%
+        checks += 1
+
+        if original.get('parent_name'):
+            # Then we start comparing parents' data
+            if candidate.get('parent_name'):
+                score += SequenceMatcher(
+                    None, original['parent_name'], candidate.get('parent_name') or ''
+                ).ratio()  # * 0.2  # 20%
+                checks += 1
+
+                score += self.__calculate_dict_diff(
+                    original['parent_attribs'], candidate.get('parent_attribs') or {}
+                )  # * 0.2  # 20%
+                checks += 1
+
+                if original['parent_text']:
+                    score += SequenceMatcher(
+                        None, original['parent_text'], candidate.get('parent_text') or ''
+                    ).ratio()  # * 0.1  # 10%
+                    checks += 1
+            # else:
+            #     # The original element have a parent and this one not, this is not a good sign
+            #     score -= 0.1
+
+        if original.get('siblings'):
+            score += SequenceMatcher(
+                None, original['siblings'], candidate.get('siblings') or []
+            ).ratio()  # * 0.1  # 10%
+            checks += 1
+
+        # How % sure? let's see
+        return round((score / checks) * 100, 2)
+
+    @staticmethod
+    def __calculate_dict_diff(dict1: dict, dict2: dict) -> float:
+        """Used internally calculate similarity between two dictionaries as SequenceMatcher doesn't accept dictionaries
+        """
+        score = SequenceMatcher(None, tuple(dict1.keys()), tuple(dict2.keys())).ratio() * 0.5
+        score += SequenceMatcher(None, tuple(dict1.values()), tuple(dict2.values())).ratio() * 0.5
+        return score
+
+    def save(self, element: Union['Adaptor', html.HtmlElement], identifier: str) -> None:
+        """Saves the element's unique properties to the storage for retrieval and relocation later
+
+        :param element: The element itself that we want to save to storage, it can be a `Adaptor` or pure `HtmlElement`
+        :param identifier: This is the identifier that will be used to retrieve the element later from the storage. See
+            the docs for more info.
+        """
+        if self.__auto_match_enabled:
+            if isinstance(element, self.__class__):
+                element = element._root
+
+            if self._is_text_node(element):
+                element = element.getparent()
+
+            self._storage.save(element, identifier)
+        else:
+            logging.critical(
+                "Can't use Auto-match features with disabled globally, you have to start a new class instance."
+            )
+
+    def retrieve(self, identifier: str) -> Optional[Dict]:
+        """Using the identifier, we search the storage and return the unique properties of the element
+
+        :param identifier: This is the identifier that will be used to retrieve the element from the storage. See
+            the docs for more info.
+        :return: A dictionary of the unique properties
+        """
+        if self.__auto_match_enabled:
+            return self._storage.retrieve(identifier)
+
+        logging.critical(
+            "Can't use Auto-match features with disabled globally, you have to start a new class instance."
+        )
+
+    # Operations on text functions
+    def json(self) -> Dict:
+        """Return json response if the response is jsonable otherwise throws error"""
+        return self.text.json()
+
+    def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True) -> 'List[str]':
+        """Apply the given regex to the current text and return a list of strings with the matches.
+
+        :param regex: Can be either a compiled regular expression or a string.
+        :param replace_entities: if enabled character entity references are replaced by their corresponding character
+        """
+        return self.text.re(regex, replace_entities)
+
+    def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True):
+        """Apply the given regex to text and return the first match if found, otherwise return the default value.
+
+        :param regex: Can be either a compiled regular expression or a string.
+        :param default: The default value to be returned if there is no match
+        :param replace_entities: if enabled character entity references are replaced by their corresponding character
+
+        """
+        return self.text.re_first(regex, default, replace_entities)
+
+    def find_similar(
+            self,
+            similarity_threshold: float = 0.2,
+            ignore_attributes: Union[List, Tuple] = ('href', 'src',),
+            match_text: bool = False
+    ) -> Union['Adaptors[Adaptor]', List]:
+        """Find elements that are in the same tree depth in the page with the same tag name and same parent tag etc...
+        then return the ones that match the current element attributes with percentage higher than the input threshold.
+
+        This function is inspired by AutoScraper and made for cases where you, for example, found a product div inside
+        a products-list container and want to find other products using that that element as a starting point EXCEPT
+        this function works in any case without depending on the element type.
+
+        :param similarity_threshold: The percentage to use while comparing elements attributes.
+            Note: Elements found before attributes matching/comparison will be sharing the same depth, same tag name,
+            same parent tag name, and same grand parent tag name. So they are 99% likely to be correct unless your are
+            extremely unlucky then attributes matching comes into play so basically don't play with this number unless
+            you are getting the results you don't want.
+            Also, if current element doesn't have attributes and the similar element as well, then it's a 100% match.
+        :param ignore_attributes: Attribute names passed will be ignored while matching the attributes in last step.
+            The default value is to ignore `href` and `src` as URLs can change a lot between elements so it's unreliable
+        :param match_text: If True, elements text content will be taken into calculation while matching.
+            Not recommended to use in normal cases but it depends.
+
+        :return: A ``Adaptors`` container of ``Adaptor`` objects or empty list
+        """
+        def get_attributes(element: html.HtmlElement) -> Dict:
+            """Return attributes dictionary without the ignored list"""
+            return {k: v for k, v in element.attrib.items() if k not in ignore_attributes}
+
+        def are_alike(original: html.HtmlElement, original_attributes: Dict, candidate: html.HtmlElement) -> bool:
+            """Calculate a score of how much these elements are alike and return True
+                if score is higher or equal the threshold"""
+            candidate_attributes = get_attributes(candidate) if ignore_attributes else candidate.attrib
+            score, checks = 0, 0
+
+            if original_attributes:
+                score += sum(
+                    SequenceMatcher(None, v, candidate_attributes.get(k, '')).ratio()
+                    for k, v in original_attributes.items()
+                )
+                checks += len(candidate_attributes)
+            else:
+                if not candidate_attributes:
+                    # Both doesn't have attributes, this must mean something
+                    score += 1
+                    checks += 1
+
+            if match_text:
+                score += SequenceMatcher(
+                    None, clean_spaces(original.text or ''), clean_spaces(candidate.text or '')
+                ).ratio()
+                checks += 1
+
+            if checks:
+                return round(score / checks, 2) >= similarity_threshold
+            return False
+
+        # We will use the elements root from now on to get the speed boost of using Lxml directly
+        root = self._root
+        current_depth = len(list(root.iterancestors()))
+        target_attrs = get_attributes(root) if ignore_attributes else root.attrib
+        similar_elements = list()
+        # + root.xpath(f"//{self.tag}[count(ancestor::*) = {current_depth-1}]")
+        parent = root.getparent()
+        if parent is not None:
+            grandparent = parent.getparent()  # lol
+            if grandparent is not None:
+                potential_matches = root.xpath(
+                    f"//{grandparent.tag}/{parent.tag}/{self.tag}[count(ancestor::*) = {current_depth}]"
+                )
+            else:
+                potential_matches = root.xpath(f"//{parent.tag}/{self.tag}[count(ancestor::*) = {current_depth}]")
+        else:
+            potential_matches = root.xpath(f"//{self.tag}[count(ancestor::*) = {current_depth}]")
+
+        for potential_match in potential_matches:
+            if potential_match != root and are_alike(root, target_attrs, potential_match):
+                similar_elements.append(potential_match)
+
+        return self.__convert_results(similar_elements)
+
+    def find_by_text(
+            self, text: str, first_match: bool = True, partial: bool = False,
+            case_sensitive: bool = False, clean_match: bool = True
+    ) -> Union['Adaptors[Adaptor]', 'Adaptor', List]:
+        """Find elements that its text content fully/partially matches input.
+        :param text: Text query to match
+        :param first_match: Return first element that matches conditions, enabled by default
+        :param partial: If enabled, function return elements that contains the input text
+        :param case_sensitive: if enabled, letters case will be taken into consideration
+        :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
+        """
+
+        results = []
+        if not case_sensitive:
+            text = text.lower()
+
+        def _traverse(node: Adaptor) -> None:
+            """Check if element matches given text otherwise, traverse the children tree and iterate"""
+            node_text = node.text
+            # if there's already no text in this node, dodge it to save CPU cycles and time
+            if node_text:
+                if clean_match:
+                    node_text = node_text.clean()
+
+                if not case_sensitive:
+                    node_text = node_text.lower()
+
+                if partial:
+                    if text in node_text:
+                        results.append(node)
+                elif text == node_text:
+                    results.append(node)
+
+            if results and first_match:
+                # we got an element so we should stop
+                return
+
+            for branch in node.children:
+                _traverse(branch)
+
+        # This will block until we traverse all children/branches
+        _traverse(self)
+
+        if first_match:
+            if results:
+                return results[0]
+        return self.__convert_results(results)
+
+    def find_by_regex(
+            self, query: str, first_match: bool = True, case_sensitive: bool = False, clean_match: bool = True
+    ) -> Union['Adaptors[Adaptor]', 'Adaptor', List]:
+        """Find elements that its text content matches the input regex pattern.
+        :param query: Regex query to match
+        :param first_match: Return first element that matches conditions, enabled by default
+        :param case_sensitive: if enabled, letters case will be taken into consideration in the regex
+        :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
+        """
+        results = []
+
+        def _traverse(node: Adaptor) -> None:
+            """Check if element matches given regex otherwise, traverse the children tree and iterate"""
+            node_text = node.text
+            # if there's already no text in this node, dodge it to save CPU cycles and time
+            if node_text:
+                if node_text.re(query, check_match=True, clean_match=clean_match, case_sensitive=case_sensitive):
+                    results.append(node)
+
+            if results and first_match:
+                # we got an element so we should stop
+                return
+
+            for branch in node.children:
+                _traverse(branch)
+
+        # This will block until we traverse all children/branches
+        _traverse(self)
+
+        if results and first_match:
+            return results[0]
+        return self.__convert_results(results)
+
+
+class Adaptors(List[Adaptor]):
+    """
+    The :class:`Adaptors` class is a subclass of the builtin ``List`` class, which provides a few additional methods.
+    """
+    __slots__ = ()
+
+    def __getitem__(self, pos: Union[SupportsIndex, slice]) -> Union[Adaptor, "Adaptors[Adaptor]"]:
+        lst = super().__getitem__(pos)
+        if isinstance(pos, slice):
+            return self.__class__(lst)
+        else:
+            return lst
+
+    def xpath(
+            self, selector: str, identifier: str = '', auto_save: bool = False, percentage: int = 0, **kwargs: Any
+    ) -> Union["Adaptors[Adaptor]", List]:
+        """
+        Call the ``.xpath()`` method for each element in this list and return
+        their results as another :class:`Adaptors`.
+
+        **Important:
+        It's recommended to use the identifier argument if you plan to use different selector later
+        and want to relocate the same element(s)**
+
+         Note: **Additional keyword arguments will be passed as XPath variables in the XPath expression!**
+
+        :param selector: The XPath selector to be used.
+        :param identifier: A string that will be used to retrieve element's data in auto-matching
+         otherwise the selector will be used.
+        :param auto_save: Automatically save new elements for `auto_match` later
+        :param percentage: The minimum percentage to accept while auto-matching and not going lower than that.
+         Be aware that the percentage calculation depends solely on the page structure so don't play with this
+         number unless you must know what you are doing!
+
+        :return: List as :class:`Adaptors`
+        """
+        results = [
+            n.xpath(selector, identifier or selector, False, auto_save, percentage, **kwargs) for n in self
+        ]
+        return self.__class__(flatten(results))
+
+    def css(self, selector: str, identifier: str = '', auto_save: bool = False, percentage: int = 0) -> Union["Adaptors[Adaptor]", List]:
+        """
+        Call the ``.css()`` method for each element in this list and return
+        their results flattened as another :class:`Adaptors`.
+
+        **Important:
+        It's recommended to use the identifier argument if you plan to use different selector later
+        and want to relocate the same element(s)**
+
+        :param selector: The CSS3 selector to be used.
+        :param identifier: A string that will be used to retrieve element's data in auto-matching
+         otherwise the selector will be used.
+        :param auto_save: Automatically save new elements for `auto_match` later
+        :param percentage: The minimum percentage to accept while auto-matching and not going lower than that.
+         Be aware that the percentage calculation depends solely on the page structure so don't play with this
+         number unless you must know what you are doing!
+
+        :return: List as :class:`Adaptors`
+        """
+        results = [
+            n.css(selector, identifier or selector, False, auto_save, percentage) for n in self
+        ]
+        return self.__class__(flatten(results))
+
+    def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True) -> 'List[str]':
+        """Call the ``.re()`` method for each element in this list and return
+        their results flattened as List of TextHandler.
+
+        :param regex: Can be either a compiled regular expression or a string.
+        :param replace_entities: if enabled character entity references are replaced by their corresponding character
+        """
+        results = [
+            n.text.re(regex, replace_entities) for n in self
+        ]
+        return flatten(results)
+
+    def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True):
+        """Call the ``.re_first()`` method for each element in this list and return
+        their results flattened as List of TextHandler.
+
+        :param regex: Can be either a compiled regular expression or a string.
+        :param default: The default value to be returned if there is no match
+        :param replace_entities: if enabled character entity references are replaced by their corresponding character
+
+        """
+        results = [
+            n.text.re_first(regex, default, replace_entities) for n in self
+        ]
+        return flatten(results)
+
+    # def __getattr__(self, name):
+    #     if name in dir(self.__class__):
+    #         return super().__getattribute__(name)
+    #
+    #     # Execute the method itself on each Adaptor
+    #     results = []
+    #     for item in self:
+    #         results.append(getattr(item, name))
+    #
+    #     if all(callable(r) for r in results):
+    #         def call_all(*args, **kwargs):
+    #             final_results = [r(*args, **kwargs) for r in results]
+    #             if all([isinstance(r, (Adaptor, Adaptors,)) for r in results]):
+    #                 return self.__class__(final_results)
+    #             return final_results
+    #
+    #         return call_all
+    #     else:
+    #         # Flatten the result if it's a single-item list containing a list
+    #         if len(self) == 1 and isinstance(results[0], list):
+    #             return self.__class__(results[0])
+    #         return self.__class__(results)
+
+    def get(self, default=None):
+        """Returns the first item of the current list
+        :param default: the default value to return if the current list is empty
+        """
+        return self[0] if len(self) > 0 else default
+
+    @property
+    def first(self):
+        """Returns the first item of the current list or `None` if the list is empty"""
+        return self.get()
+
+    @property
+    def last(self):
+        """Returns the last item of the current list or `None` if the list is empty"""
+        return self[-1] if len(self) > 0 else None
+
+    def __getstate__(self) -> Any:
+        # lxml don't like it :)
+        raise TypeError("Can't pickle Adaptors object")

scrapling/storage_adaptors.py

@@ -0,0 +1,149 @@
+import orjson
+import sqlite3
+import logging
+import threading
+from hashlib import sha256
+from abc import ABC, abstractmethod
+from typing import Dict, Optional, Union
+
+from scrapling.utils import _StorageTools, cache
+
+from lxml import html
+from tldextract import extract as tld
+
+
+class StorageSystemMixin(ABC):
+    # If you want to make your own storage system, you have to inherit from this
+    def __init__(self, url: Union[str, None] = None):
+        """
+        :param url: URL of the website we are working on to separate it from other websites data
+        """
+        self.url = url
+
+    @cache
+    def _get_base_url(self, default_value: str = 'default') -> str:
+        if not self.url or type(self.url) is not str:
+            return default_value
+
+        try:
+            extracted = tld(self.url)
+            return extracted.registered_domain or extracted.domain or default_value
+        except AttributeError:
+            return default_value
+
+    @abstractmethod
+    def save(self, element: html.HtmlElement, identifier: str) -> None:
+        """Saves the element's unique properties to the storage for retrieval and relocation later
+
+        :param element: The element itself that we want to save to storage.
+        :param identifier: This is the identifier that will be used to retrieve the element later from the storage. See
+            the docs for more info.
+        """
+        raise NotImplementedError('Storage system must implement `save` method')
+
+    @abstractmethod
+    def retrieve(self, identifier: str) -> Optional[Dict]:
+        """Using the identifier, we search the storage and return the unique properties of the element
+
+        :param identifier: This is the identifier that will be used to retrieve the element from the storage. See
+            the docs for more info.
+        :return: A dictionary of the unique properties
+        """
+        raise NotImplementedError('Storage system must implement `save` method')
+
+    @staticmethod
+    @cache
+    def _get_hash(identifier: str) -> str:
+        """If you want to hash identifier in your storage system, use this safer"""
+        identifier = identifier.lower().strip()
+        if isinstance(identifier, str):
+            # Hash functions have to take bytes
+            identifier = identifier.encode('utf-8')
+
+        hash_value = sha256(identifier).hexdigest()
+        return f"{hash_value}_{len(identifier)}"  # Length to reduce collision chance
+
+
+@cache(None, typed=True)
+class SQLiteStorageSystem(StorageSystemMixin):
+    """The recommended system to use, it's race condition safe and thread safe.
+    Mainly built so the library can run in threaded frameworks like scrapy or threaded tools
+    > It's optimized for threaded applications but running it without threads shouldn't make it slow."""
+    def __init__(self, storage_file: str, url: Union[str, None] = None):
+        """
+        :param storage_file: File to be used to store elements
+        :param url: URL of the website we are working on to separate it from other websites data
+
+        """
+        super().__init__(url)
+        self.storage_file = storage_file
+        # We use a threading.Lock to ensure thread-safety instead of relying on thread-local storage.
+        self.lock = threading.Lock()
+        # >SQLite default mode in earlier version is 1 not 2 (1=thread-safe 2=serialized)
+        # `check_same_thread=False` to allow it to be used across different threads.
+        self.connection = sqlite3.connect(self.storage_file, check_same_thread=False)
+        # WAL (Write-Ahead Logging) allows for better concurrency.
+        self.connection.execute("PRAGMA journal_mode=WAL")
+        self.cursor = self.connection.cursor()
+        self._setup_database()
+        logging.debug(
+            f'Storage system loaded with arguments (storage_file="{storage_file}", url="{url}")'
+        )
+
+    def _setup_database(self) -> None:
+        self.cursor.execute("""
+            CREATE TABLE IF NOT EXISTS storage (
+                id INTEGER PRIMARY KEY,
+                url TEXT,
+                identifier TEXT,
+                element_data TEXT,
+                UNIQUE (url, identifier)
+            )
+        """)
+        self.connection.commit()
+
+    def save(self, element: html.HtmlElement, identifier: str):
+        """Saves the elements unique properties to the storage for retrieval and relocation later
+
+        :param element: The element itself that we want to save to storage.
+        :param identifier: This is the identifier that will be used to retrieve the element later from the storage. See
+            the docs for more info.
+        """
+        url = self._get_base_url()
+        element_data = _StorageTools.element_to_dict(element)
+        with self.lock:
+            self.cursor.execute("""
+                INSERT OR REPLACE INTO storage (url, identifier, element_data)
+                VALUES (?, ?, ?)
+            """, (url, identifier, orjson.dumps(element_data)))
+            self.cursor.fetchall()
+            self.connection.commit()
+
+    def retrieve(self, identifier: str) -> Optional[Dict]:
+        """Using the identifier, we search the storage and return the unique properties of the element
+
+        :param identifier: This is the identifier that will be used to retrieve the element from the storage. See
+            the docs for more info.
+        :return: A dictionary of the unique properties
+        """
+        url = self._get_base_url()
+        with self.lock:
+            self.cursor.execute(
+                "SELECT element_data FROM storage WHERE url = ? AND identifier = ?",
+                (url, identifier)
+            )
+            result = self.cursor.fetchone()
+            if result:
+                return orjson.loads(result[0])
+            return None
+
+    def close(self):
+        """Close all connections, will be useful when with some things like scrapy Spider.closed() function/signal"""
+        with self.lock:
+            self.connection.commit()
+            self.cursor.close()
+            self.connection.close()
+
+    def __del__(self):
+        """To ensure all connections are closed when the object is destroyed."""
+        self.close()

scrapling/translator.py

@@ -0,0 +1,148 @@
+"""
+Most of this file is adapted version of the translator of parsel library with some modifications simply for 1 important reason...
+To add pseudo-elements ``::text`` and ``::attr(ATTR_NAME)`` so we match Parsel/Scrapy selectors format
+which will be important in future releases but most importantly...
+    so you don't have to learn a new selectors/api method like what bs4 done with soupsieve :)
+> if you want to learn about this, head to https://cssselect.readthedocs.io/en/latest/#cssselect.FunctionalPseudoElement
+"""
+
+import re
+
+from w3lib.html import HTML5_WHITESPACE
+from typing import TYPE_CHECKING, Any, Optional, Protocol
+
+from scrapling.utils import cache
+
+from cssselect.xpath import ExpressionError
+from cssselect.xpath import XPathExpr as OriginalXPathExpr
+from cssselect import HTMLTranslator as OriginalHTMLTranslator
+from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement
+
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
+
+regex = f"[{HTML5_WHITESPACE}]+"
+replace_html5_whitespaces = re.compile(regex).sub
+
+
+class XPathExpr(OriginalXPathExpr):
+
+    textnode: bool = False
+    attribute: Optional[str] = None
+
+    @classmethod
+    def from_xpath(
+        cls,
+        xpath: OriginalXPathExpr,
+        textnode: bool = False,
+        attribute: Optional[str] = None,
+    ) -> "Self":
+        x = cls(path=xpath.path, element=xpath.element, condition=xpath.condition)
+        x.textnode = textnode
+        x.attribute = attribute
+        return x
+
+    def __str__(self) -> str:
+        path = super().__str__()
+        if self.textnode:
+            if path == "*":
+                path = "text()"
+            elif path.endswith("::*/*"):
+                path = path[:-3] + "text()"
+            else:
+                path += "/text()"
+
+        if self.attribute is not None:
+            if path.endswith("::*/*"):
+                path = path[:-2]
+            path += f"/@{self.attribute}"
+
+        return path
+
+    def join(
+        self: "Self",
+        combiner: str,
+        other: OriginalXPathExpr,
+        *args: Any,
+        **kwargs: Any,
+    ) -> "Self":
+        if not isinstance(other, XPathExpr):
+            raise ValueError(
+                f"Expressions of type {__name__}.XPathExpr can ony join expressions"
+                f" of the same type (or its descendants), got {type(other)}"
+            )
+        super().join(combiner, other, *args, **kwargs)
+        self.textnode = other.textnode
+        self.attribute = other.attribute
+        return self
+
+
+# e.g. cssselect.GenericTranslator, cssselect.HTMLTranslator
+class TranslatorProtocol(Protocol):
+    def xpath_element(self, selector: Element) -> OriginalXPathExpr:
+        pass
+
+    def css_to_xpath(self, css: str, prefix: str = ...) -> str:
+        pass
+
+
+class TranslatorMixin:
+    """This mixin adds support to CSS pseudo elements via dynamic dispatch.
+
+    Currently supported pseudo-elements are ``::text`` and ``::attr(ATTR_NAME)``.
+    """
+
+    def xpath_element(self: TranslatorProtocol, selector: Element) -> XPathExpr:
+        # https://github.com/python/mypy/issues/12344
+        xpath = super().xpath_element(selector)  # type: ignore[safe-super]
+        return XPathExpr.from_xpath(xpath)
+
+    def xpath_pseudo_element(
+        self, xpath: OriginalXPathExpr, pseudo_element: PseudoElement
+    ) -> OriginalXPathExpr:
+        """
+        Dispatch method that transforms XPath to support pseudo-elements.
+        """
+        if isinstance(pseudo_element, FunctionalPseudoElement):
+            method_name = f"xpath_{pseudo_element.name.replace('-', '_')}_functional_pseudo_element"
+            method = getattr(self, method_name, None)
+            if not method:
+                raise ExpressionError(
+                    f"The functional pseudo-element ::{pseudo_element.name}() is unknown"
+                )
+            xpath = method(xpath, pseudo_element)
+        else:
+            method_name = (
+                f"xpath_{pseudo_element.replace('-', '_')}_simple_pseudo_element"
+            )
+            method = getattr(self, method_name, None)
+            if not method:
+                raise ExpressionError(
+                    f"The pseudo-element ::{pseudo_element} is unknown"
+                )
+            xpath = method(xpath)
+        return xpath
+
+    @staticmethod
+    def xpath_attr_functional_pseudo_element(
+            xpath: OriginalXPathExpr, function: FunctionalPseudoElement
+    ) -> XPathExpr:
+        """Support selecting attribute values using ::attr() pseudo-element"""
+        if function.argument_types() not in (["STRING"], ["IDENT"]):
+            raise ExpressionError(
+                f"Expected a single string or ident for ::attr(), got {function.arguments!r}"
+            )
+        return XPathExpr.from_xpath(xpath, attribute=function.arguments[0].value)
+
+    @staticmethod
+    def xpath_text_simple_pseudo_element(xpath: OriginalXPathExpr) -> XPathExpr:
+        """Support selecting text nodes using ::text pseudo-element"""
+        return XPathExpr.from_xpath(xpath, textnode=True)
+
+
+class HTMLTranslator(TranslatorMixin, OriginalHTMLTranslator):
+    @cache(maxsize=256)
+    def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str:
+        return super().css_to_xpath(css, prefix)

scrapling/utils.py

@@ -0,0 +1,164 @@
+import re
+import os
+import logging
+from itertools import chain
+from logging import handlers
+# Using cache on top of a class is brilliant way to achieve Singleton design pattern without much code
+from functools import lru_cache as cache  # functools.cache is available on Python 3.9+ only so let's keep lru_cache
+
+from typing import Dict, Iterable, Any
+
+from lxml import html
+html_forbidden = {html.HtmlComment, }
+logging.basicConfig(
+        level=logging.ERROR,
+        format='%(asctime)s - %(levelname)s - %(message)s',
+        handlers=[
+            logging.StreamHandler()
+        ]
+    )
+
+
+@cache(None, typed=True)
+def setup_basic_logging(level: str = 'debug'):
+    levels = {
+        'debug': logging.DEBUG,
+        'info': logging.INFO,
+        'warning': logging.WARNING,
+        'error': logging.ERROR,
+        'critical': logging.CRITICAL
+    }
+    formatter = logging.Formatter("[%(asctime)s] %(levelname)s: %(message)s", "%Y-%m-%d %H:%M:%S")
+    lvl = levels[level.lower()]
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+    # Configure the root logger
+    logging.basicConfig(level=lvl, handlers=[handler])
+
+
+def flatten(lst: Iterable):
+    return list(chain.from_iterable(lst))
+
+
+def _is_iterable(s: Any):
+    # This will be used only in regex functions to make sure it's iterable but not string/bytes
+    return isinstance(s, (list, tuple,))
+
+
+@cache(None, typed=True)
+class _Logger(object):
+    # I will leave this class here for now in case I decide I want to come back to use it :)
+    __slots__ = ('console_logger', 'logger_file_path',)
+    levels = {
+        'debug': logging.DEBUG,
+        'info': logging.INFO,
+        'warning': logging.WARNING,
+        'error': logging.ERROR,
+        'critical': logging.CRITICAL
+    }
+
+    def __init__(self, filename: str = 'debug.log', level: str = 'debug', when: str = 'midnight', backcount: int = 1):
+        os.makedirs(os.path.join(os.path.dirname(__file__), 'logs'), exist_ok=True)
+        format_str = logging.Formatter("[%(asctime)s] %(levelname)s: %(message)s", "%Y-%m-%d %H:%M:%S")
+
+        # on-screen output
+        lvl = self.levels[level.lower()]
+        self.console_logger = logging.getLogger('Scrapling')
+        self.console_logger.setLevel(lvl)
+        console_handler = logging.StreamHandler()
+        console_handler.setLevel(lvl)
+        console_handler.setFormatter(format_str)
+        self.console_logger.addHandler(console_handler)
+
+        if lvl == logging.DEBUG:
+            filename = os.path.join(os.path.dirname(__file__), 'logs', filename)
+            self.logger_file_path = filename
+            # Automatically generates the logging file at specified intervals
+            file_handler = handlers.TimedRotatingFileHandler(
+                # If more than (backcount+1) existed, oldest logs will be deleted
+                filename=filename, when=when, backupCount=backcount, encoding='utf-8'
+            )
+            file_handler.setLevel(lvl)
+            file_handler.setFormatter(format_str)
+            # This for the logger when it appends the date to the new log
+            file_handler.namer = lambda name: name.replace(".log", "") + ".log"
+            self.console_logger.addHandler(file_handler)
+            self.debug(f'Debug log path: {self.logger_file_path}')
+        else:
+            self.logger_file_path = None
+
+    def debug(self, message: str) -> None:
+        self.console_logger.debug(message)
+
+    def info(self, message: str) -> None:
+        self.console_logger.info(message)
+
+    def warning(self, message: str) -> None:
+        self.console_logger.warning(message)
+
+    def error(self, message: str) -> None:
+        self.console_logger.error(message)
+
+    def critical(self, message: str) -> None:
+        self.console_logger.critical(message)
+
+
+class _StorageTools:
+    @staticmethod
+    def __clean_attributes(element: html.HtmlElement, forbidden: tuple = ()) -> Dict:
+        if not element.attrib:
+            return {}
+        return {k: v.strip() for k, v in element.attrib.items() if v and v.strip() and k not in forbidden}
+
+    @classmethod
+    def element_to_dict(cls, element: html.HtmlElement) -> Dict:
+        parent = element.getparent()
+        result = {
+            'tag': str(element.tag),
+            'attributes': cls.__clean_attributes(element),
+            'text': element.text.strip() if element.text else None,
+            'path': cls._get_element_path(element)
+        }
+        if parent is not None:
+            result.update({
+                'parent_name': parent.tag,
+                'parent_attribs': dict(parent.attrib),
+                'parent_text': parent.text.strip() if parent.text else None
+            })
+
+            siblings = [child.tag for child in parent.iterchildren() if child != element]
+            if siblings:
+                result.update({'siblings': tuple(siblings)})
+
+        children = [child.tag for child in element.iterchildren() if type(child) not in html_forbidden]
+        if children:
+            result.update({'children': tuple(children)})
+
+        return result
+
+    @classmethod
+    def _get_element_path(cls, element: html.HtmlElement):
+        parent = element.getparent()
+        return tuple(
+            (element.tag,) if parent is None else (
+                    cls._get_element_path(parent) + (element.tag,)
+            )
+        )
+
+
+# def _root_type_verifier(method):
+#     # Just to make sure we are safe
+#     @wraps(method)
+#     def _impl(self, *args, **kw):
+#         # All html types inherits from HtmlMixin so this to check for all at once
+#         if not issubclass(type(self._root), html.HtmlMixin):
+#             raise ValueError(f"Cannot use function on a Node of type {type(self._root)!r}")
+#         return method(self, *args, **kw)
+#     return _impl
+
+
+@cache
+def clean_spaces(string):
+    string = string.replace('\t', ' ')
+    string = re.sub('[\n|\r]', '', string)
+    return re.sub(' +', ' ', string)

setup.cfg

@@ -0,0 +1,8 @@
+[metadata]
+name = scrapling
+version = 0.1
+author = Karim Shoair
+author_email = karim.shoair@pm.me
+description = Scrapling is a powerful, flexible, adaptive, and high-performance web scraping library for Python.
+license = BSD
+home-page = https://github.com/D4Vinci/Scrapling

setup.py

@@ -0,0 +1,65 @@
+from setuptools import setup
+
+with open("README.md", "r", encoding="utf-8") as fh:
+    long_description = fh.read()
+
+
+setup(
+    name="scrapling",
+    version="0.1",
+    description="""Scrapling is a powerful, flexible, and high-performance web scraping library for Python. It 
+    simplifies the process of extracting data from websites, even when they undergo structural changes, and offers 
+    impressive speed improvements over many popular scraping tools.""",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    author="Karim Shoair",
+    author_email="karim.shoair@pm.me",
+    license="BSD",
+    packages=["scrapling",],
+    zip_safe=False,
+    package_dir={
+        "scrapling": "scrapling",
+    },
+    include_package_data=True,
+    classifiers=[
+        "Operating System :: OS Independent",
+        "Development Status :: 4 - Beta ",
+        # "Development Status :: 5 - Production/Stable",
+        # "Development Status :: 6 - Mature",
+        # "Development Status :: 7 - Inactive",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: BSD License",
+        "Natural Language :: English",
+        "Topic :: Internet :: WWW/HTTP",
+        "Topic :: Text Processing :: Markup",
+        "Topic :: Text Processing :: Markup :: HTML",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3 :: Only",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: Implementation :: CPython",
+        "Typing :: Typed",
+    ],
+    # Instead of using requirements file to dodge possible errors from tox?
+    install_requires=[
+        "requests>=2.3",
+        "lxml>=4.5",
+        "cssselect>=1.2",
+        "w3lib",
+        "orjson>=3",
+        "tldextract",
+    ],
+    python_requires=">=3.6",
+    url="https://github.com/D4Vinci/Scrapling",
+    project_urls={
+        "Documentation": "https://github.com/D4Vinci/Scrapling/Docs",  # For now
+        "Source": "https://github.com/D4Vinci/Scrapling",
+        "Tracker": "https://github.com/D4Vinci/Scrapling/issues",
+    }
+)

tests/__init__.py

@@ -0,0 +1 @@
+"""Package for test project."""

tests/requirements.txt

@@ -0,0 +1,2 @@
+pytest
+pytest-cov

tests/test_all_functions.py

@@ -0,0 +1,336 @@
+
+import pickle
+import unittest
+from scrapling import Adaptor
+from cssselect import SelectorError, SelectorSyntaxError
+
+
+class TestParser(unittest.TestCase):
+    def setUp(self):
+        self.html = '''
+        <html>
+        <head>
+            <title>Complex Web Page</title>
+            <style>
+                .hidden { display: none; }
+            </style>
+        </head>
+        <body>
+            <header>
+                <nav>
+                    <ul>
+                        <li><a href="#home">Home</a></li>
+                        <li><a href="#about">About</a></li>
+                        <li><a href="#contact">Contact</a></li>
+                    </ul>
+                </nav>
+            </header>
+            <main>
+                <section id="products" schema='{"jsonable": "data"}'>
+                    <h2>Products</h2>
+                    <div class="product-list">
+                        <article class="product" data-id="1">
+                            <h3>Product 1</h3>
+                            <p class="description">This is product 1</p>
+                            <span class="price">$10.99</span>
+                            <div class="hidden stock">In stock: 5</div>
+                        </article>
+                        <article class="product" data-id="2">
+                            <h3>Product 2</h3>
+                            <p class="description">This is product 2</p>
+                            <span class="price">$20.99</span>
+                            <div class="hidden stock">In stock: 3</div>
+                        </article>
+                        <article class="product" data-id="3">
+                            <h3>Product 3</h3>
+                            <p class="description">This is product 3</p>
+                            <span class="price">$15.99</span>
+                            <div class="hidden stock">Out of stock</div>
+                        </article>
+                    </div>
+                </section>
+                <section id="reviews">
+                    <h2>Customer Reviews</h2>
+                    <div class="review-list">
+                        <div class="review" data-rating="5">
+                            <p class="review-text">Great product!</p>
+                            <span class="reviewer">John Doe</span>
+                        </div>
+                        <div class="review" data-rating="4">
+                            <p class="review-text">Good value for money.</p>
+                            <span class="reviewer">Jane Smith</span>
+                        </div>
+                    </div>
+                </section>
+            </main>
+            <footer>
+                <p>&copy; 2024 Our Company</p>
+            </footer>
+            <script id="page-data" type="application/json">
+                {"lastUpdated": "2024-09-22T10:30:00Z", "totalProducts": 3}
+            </script>
+        </body>
+        </html>
+        '''
+        self.page = Adaptor(self.html, auto_match=False, debug=False)
+
+    def test_css_selector(self):
+        """Test Selecting elements with complex CSS selectors"""
+        elements = self.page.css('main #products .product-list article.product')
+        self.assertEqual(len(elements), 3)
+
+        in_stock_products = self.page.css(
+            'main #products .product-list article.product:not(:contains("Out of stock"))')
+        self.assertEqual(len(in_stock_products), 2)
+
+    def test_xpath_selector(self):
+        """Test Selecting elements with Complex XPath selectors"""
+        reviews = self.page.xpath(
+            '//section[@id="reviews"]//div[contains(@class, "review") and @data-rating >= 4]'
+        )
+        self.assertEqual(len(reviews), 2)
+
+        high_priced_products = self.page.xpath(
+            '//article[contains(@class, "product")]'
+            '[number(translate(substring-after(.//span[@class="price"], "$"), ",", "")) > 15]'
+        )
+        self.assertEqual(len(high_priced_products), 2)
+
+    def test_find_by_text(self):
+        """Test Selecting elements with Text matching"""
+        stock_info = self.page.find_by_regex(r'In stock: \d+', first_match=False)
+        self.assertEqual(len(stock_info), 2)
+
+        stock_info = self.page.find_by_regex(r'In stock: \d+', first_match=True, case_sensitive=True)
+        self.assertEqual(stock_info.text, 'In stock: 5')
+
+        stock_info = self.page.find_by_text(r'In stock:', partial=True, first_match=False)
+        self.assertEqual(len(stock_info), 2)
+
+        out_of_stock = self.page.find_by_text('Out of stock', partial=False, first_match=False)
+        self.assertEqual(len(out_of_stock), 1)
+
+    def test_find_similar_elements(self):
+        """Test Finding similar elements of an element"""
+        first_product = self.page.css('.product')[0]
+        similar_products = first_product.find_similar()
+        self.assertEqual(len(similar_products), 2)
+
+        first_review = self.page.css('.review')[0]
+        similar_high_rated_reviews = [
+            review
+            for review in first_review.find_similar()
+            if int(review.attrib.get('data-rating', 0)) >= 4
+        ]
+        self.assertEqual(len(similar_high_rated_reviews), 1)
+
+    def test_expected_errors(self):
+        """Test errors that should raised if it does"""
+        with self.assertRaises(ValueError):
+            _ = Adaptor()
+
+        with self.assertRaises(TypeError):
+            _ = Adaptor(root="ayo")
+
+        with self.assertRaises(TypeError):
+            _ = Adaptor(text=1)
+
+        with self.assertRaises(TypeError):
+            _ = Adaptor(body=1)
+
+        with self.assertRaises(ValueError):
+            _ = Adaptor(self.html, storage=object, auto_match=True)
+
+    def test_pickleable(self):
+        """Test that objects aren't pickleable"""
+        table = self.page.css('.product-list')[0]
+        with self.assertRaises(TypeError):  # Adaptors
+            pickle.dumps(table)
+
+        with self.assertRaises(TypeError):  # Adaptor
+            pickle.dumps(table[0])
+
+    def test_overridden(self):
+        """Test overridden functions"""
+        table = self.page.css('.product-list')[0]
+        self.assertTrue(issubclass(type(table.__str__()), str))
+        self.assertTrue(issubclass(type(table.__repr__()), str))
+        self.assertTrue(issubclass(type(table.attrib.__str__()), str))
+        self.assertTrue(issubclass(type(table.attrib.__repr__()), str))
+
+    def test_bad_selector(self):
+        """Test object can handle bad selector"""
+        with self.assertRaises((SelectorError, SelectorSyntaxError,)):
+            self.page.css('4 ayo')
+
+        with self.assertRaises((SelectorError, SelectorSyntaxError,)):
+            self.page.xpath('4 ayo')
+
+    def test_selectors_generation(self):
+        """Try to create selectors for all elements in the page"""
+        def _traverse(element: Adaptor):
+            self.assertTrue(type(element.css_selector) is str)
+            self.assertTrue(type(element.xpath_selector) is str)
+            for branch in element.children:
+                _traverse(branch)
+
+        _traverse(self.page)
+
+    def test_getting_all_text(self):
+        """Test getting all text"""
+        self.assertNotEqual(self.page.get_all_text(), '')
+
+    def test_element_navigation(self):
+        """Test moving in the page from selected element"""
+        table = self.page.css('.product-list')[0]
+
+        self.assertIsNot(table.path, [])
+        self.assertNotEqual(table.html_content, '')
+        self.assertNotEqual(table.prettify(), '')
+
+        parent = table.parent
+        self.assertEqual(parent.attrib['id'], 'products')
+
+        children = table.children
+        self.assertEqual(len(children), 3)
+
+        parent_siblings = parent.siblings
+        self.assertEqual(len(parent_siblings), 1)
+
+        child = table.css('[data-id="1"]')[0]
+        next_element = child.next
+        self.assertEqual(next_element.attrib['data-id'], '2')
+
+        prev_element = next_element.previous
+        self.assertEqual(prev_element.tag, child.tag)
+
+        all_prices = self.page.css('.price')
+        products_with_prices = [
+            price.find_ancestor(lambda p: p.has_class('product'))
+            for price in all_prices
+        ]
+        self.assertEqual(len(products_with_prices), 3)
+
+    def test_empty_return(self):
+        """Test cases where functions shouldn't have results"""
+        test_html = """
+        <html>
+            <span id="a"><a></a><!--comment--></span>
+            <span id="b"><!--comment--><a></a></span>
+        </html>"""
+        soup = Adaptor(test_html, auto_match=False, keep_comments=False)
+        html_tag = soup.css('html')[0]
+        self.assertEqual(html_tag.path, [])
+        self.assertEqual(html_tag.siblings, [])
+        self.assertEqual(html_tag.parent, None)
+        self.assertEqual(html_tag.find_ancestor(lambda e: e), None)
+
+        self.assertEqual(soup.css('#a a')[0].next, None)
+        self.assertEqual(soup.css('#b a')[0].previous, None)
+
+    def test_text_to_json(self):
+        """Test converting text to json"""
+        script_content = self.page.css('#page-data::text')[0]
+        self.assertTrue(issubclass(type(script_content.sort()), str))
+        page_data = script_content.json()
+        self.assertEqual(page_data['totalProducts'], 3)
+        self.assertTrue('lastUpdated' in page_data)
+
+    def test_regex_on_text(self):
+        """Test doing regex on a selected text"""
+        element = self.page.css('[data-id="1"] .price')[0]
+        match = element.re_first(r'[\.\d]+')
+        self.assertEqual(match, '10.99')
+        match = element.text.re(r'(\d+)', replace_entities=False)
+        self.assertEqual(len(match), 2)
+
+    def test_attribute_operations(self):
+        """Test operations on elements attributes"""
+        products = self.page.css('.product')
+        product_ids = [product.attrib['data-id'] for product in products]
+        self.assertEqual(product_ids, ['1', '2', '3'])
+        self.assertTrue('data-id' in products[0].attrib)
+
+        reviews = self.page.css('.review')
+        review_ratings = [int(review.attrib['data-rating']) for review in reviews]
+        self.assertEqual(sum(review_ratings) / len(review_ratings), 4.5)
+
+        key_value = list(products[0].attrib.search_values('1', partial=False))
+        self.assertEqual(list(key_value[0].keys()), ['data-id'])
+
+        key_value = list(products[0].attrib.search_values('1', partial=True))
+        self.assertEqual(list(key_value[0].keys()), ['data-id'])
+
+        attr_json = self.page.css('#products')[0].attrib['schema'].json()
+        self.assertEqual(attr_json, {'jsonable': 'data'})
+        self.assertEqual(type(self.page.css('#products')[0].attrib.json_string), bytes)
+
+    def test_element_relocation(self):
+        """Test relocating element after structure change"""
+        original_html = '''
+                <div class="container">
+                    <section class="products">
+                        <article class="product" id="p1">
+                            <h3>Product 1</h3>
+                            <p class="description">Description 1</p>
+                        </article>
+                        <article class="product" id="p2">
+                            <h3>Product 2</h3>
+                            <p class="description">Description 2</p>
+                        </article>
+                    </section>
+                </div>
+                '''
+        changed_html = '''
+                <div class="new-container">
+                    <div class="product-wrapper">
+                        <section class="products">
+                            <article class="product new-class" data-id="p1">
+                                <div class="product-info">
+                                    <h3>Product 1</h3>
+                                    <p class="new-description">Description 1</p>
+                                </div>
+                            </article>
+                            <article class="product new-class" data-id="p2">
+                                <div class="product-info">
+                                    <h3>Product 2</h3>
+                                    <p class="new-description">Description 2</p>
+                                </div>
+                            </article>
+                        </section>
+                    </div>
+                </div>
+                '''
+
+        old_page = Adaptor(original_html, url='example.com', auto_match=True, debug=True)
+        new_page = Adaptor(changed_html, url='example.com', auto_match=True, debug=True)
+
+        # 'p1' was used as ID and now it's not and all the path elements have changes
+        # Also at the same time testing auto-match vs combined selectors
+        _ = old_page.css('#p1, #p2', auto_save=True)[0]
+        relocated = new_page.css('#p1', auto_match=True)
+
+        self.assertIsNotNone(relocated)
+        self.assertEqual(relocated[0].attrib['data-id'], 'p1')
+        self.assertTrue(relocated[0].has_class('new-class'))
+        self.assertEqual(relocated[0].css('.new-description')[0].text, 'Description 1')
+
+    def test_performance(self):
+        """Test parsing and selecting speed"""
+        import time
+        large_html = '<html><body>' + '<div class="item">' * 5000 + '</div>' * 5000 + '</body></html>'
+
+        start_time = time.time()
+        parsed = Adaptor(large_html, auto_match=False, debug=False)
+        elements = parsed.css('.item')
+        end_time = time.time()
+
+        self.assertEqual(len(elements), 5000)
+        # Converting 5000 elements to a class and doing operations on them will take time
+        # Based on my tests with 100 runs, 1 loop each Scrapling (given the extra work/features) takes 10.4ms on average
+        self.assertLess(end_time - start_time, 0.1)
+
+
+# Use `coverage run -m unittest --verbose tests/test_all_functions.py` instead for the coverage report
+# if __name__ == '__main__':
+#     unittest.main(verbosity=2)

tox.ini

@@ -0,0 +1,20 @@
+# Tox (https://tox.readthedocs.io/) is a tool for running tests
+# in multiple virtualenvs. This configuration file will run the
+# test suite on all supported python versions. To use it, "pip install tox"
+# and then run "tox" from this directory.
+
+[tox]
+envlist = pre-commit,py36,py37,py38,py39,py310,py311,py312
+
+[testenv]
+usedevelop = True
+changedir = tests
+deps =
+    -r{toxinidir}/tests/requirements.txt
+commands = pytest --cov=scrapling --cov-report=xml
+
+[testenv:pre-commit]
+basepython = python3
+deps = pre-commit
+commands = pre-commit run --all-files --show-diff-on-failure
+skip_install = true