From 58939e10eeb7433abb2904ea36fc85f765788b9c Mon Sep 17 00:00:00 2001
From: Thibault Saunier <tsaunier@igalia.com>
Date: Sat, 18 Jan 2025 13:55:36 -0300
Subject: [PATCH] docs: Embed the gstreamer-rs documentation into our
 documentation in CI

Downloading the latest build of GStreamer-rs from its CI job and running a
script to embed the rustoc generated documentation into hotdoc based one.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/8317>
---
 ci/docker/fedora/install-deps.sh              |   2 +-
 ci/scripts/build-docs.sh                      |   3 +
 .../scripts/assets/css/rustdoc-in-hotdoc.css  |  31 +
 .../scripts/assets/js/language-menu.js        |  92 +++
 .../scripts/assets/js/sitemap-rs-fixer.js     |  76 +++
 .../gst-docs/scripts/assets/js/theme-sync.js  |  34 ++
 .../gst-docs/scripts/rust_doc_unifier.py      | 564 ++++++++++++++++++
 7 files changed, 801 insertions(+), 1 deletion(-)
 create mode 100644 subprojects/gst-docs/scripts/assets/css/rustdoc-in-hotdoc.css
 create mode 100644 subprojects/gst-docs/scripts/assets/js/language-menu.js
 create mode 100644 subprojects/gst-docs/scripts/assets/js/sitemap-rs-fixer.js
 create mode 100644 subprojects/gst-docs/scripts/assets/js/theme-sync.js
 create mode 100755 subprojects/gst-docs/scripts/rust_doc_unifier.py

diff --git a/ci/docker/fedora/install-deps.sh b/ci/docker/fedora/install-deps.sh
index 625b43c9f6..38c73407c1 100644
--- a/ci/docker/fedora/install-deps.sh
+++ b/ci/docker/fedora/install-deps.sh
@@ -28,7 +28,7 @@ dnf install -y glib2-doc gdk-pixbuf2-devel gtk3-devel-docs gtk4-devel-docs libso
 # Make sure we don't end up installing these from some transient dependency
 dnf remove -y "gstreamer1*-devel" rust cargo meson 'fdk-aac-free*'
 
-pip3 install meson==1.5.2 python-gitlab tomli junitparser
+pip3 install meson==1.5.2 python-gitlab tomli junitparser bs4
 pip3 install git+https://github.com/hotdoc/hotdoc.git@8c1cc997f5bc16e068710a8a8121f79ac25cbcce
 
 # Install most debug symbols, except the big ones from things we use
diff --git a/ci/scripts/build-docs.sh b/ci/scripts/build-docs.sh
index 6ba6cb6817..991c7011bb 100755
--- a/ci/scripts/build-docs.sh
+++ b/ci/scripts/build-docs.sh
@@ -32,3 +32,6 @@ export GI_TYPELIB_PATH=$PWD/girs
 hotdoc run --conf-file build/subprojects/gst-docs/GStreamer-doc.json
 
 mv "$builddir/subprojects/gst-docs/GStreamer-doc/html" documentation/
+
+pip3 install bs4
+python3 subprojects/gst-docs/scripts/rust_doc_unifier.py documentation/
diff --git a/subprojects/gst-docs/scripts/assets/css/rustdoc-in-hotdoc.css b/subprojects/gst-docs/scripts/assets/css/rustdoc-in-hotdoc.css
new file mode 100644
index 0000000000..5021f8e1be
--- /dev/null
+++ b/subprojects/gst-docs/scripts/assets/css/rustdoc-in-hotdoc.css
@@ -0,0 +1,31 @@
+body.rustdoc {
+  padding-top: 50px;
+}
+
+/* Revert hotdoc body font size setting */
+:root {
+  font-size: 16px;
+}
+
+/* Media queries for responsive behavior */
+@media (max-width: 767px) {
+  .gst-navbar-toggle {
+    display: block;
+  }
+}
+
+@media (max-width: 701px) {
+  .sidebar {
+    padding-top: 50px;
+  }
+}
+
+@media (min-width: 700px) {
+  .sidebar {
+    position: sticky;
+    top: 50px;
+    height: calc(100vh - 50px);
+    overflow-y: auto;
+  }
+}
+
diff --git a/subprojects/gst-docs/scripts/assets/js/language-menu.js b/subprojects/gst-docs/scripts/assets/js/language-menu.js
new file mode 100644
index 0000000000..0813c405b3
--- /dev/null
+++ b/subprojects/gst-docs/scripts/assets/js/language-menu.js
@@ -0,0 +1,92 @@
+// Wait for the language dropdown to be created
+const language_observer = new MutationObserver((mutations) => {
+    mutations.forEach((mutation) => {
+        if (mutation.addedNodes.length) {
+            const languageDropdown = document.querySelector('ul.dropdown-menu li a[href*="gi-language="]');
+            if (languageDropdown) {
+                // Found the language dropdown, so we can add Rust
+                const dropdownMenu = languageDropdown.closest('ul.dropdown-menu');
+                if (dropdownMenu && !dropdownMenu.querySelector('a[href*="rust"]')) {
+                    // Create new list item for Rust
+                    const rustItem = document.createElement('li');
+                    const rustLink = document.createElement('a');
+                    rustLink.href = 'rust/stable/latest/docs/index.html?gi-language=rust';
+                    rustLink.textContent = 'rust';
+                    rustItem.appendChild(rustLink);
+                    dropdownMenu.appendChild(rustItem);
+
+                    // Disconnect language_observer since we've done our work
+                    language_observer.disconnect();
+                }
+            }
+        }
+    });
+});
+
+
+function transformVersionMenu(versions) {
+    if (!versions) {
+        console.error("hotdoc rustdoc version could not be loaded");
+        return;
+    }
+    // Find the menu
+    const menu = document.getElementById('API versions-menu');
+    if (!menu) {
+        console.error("API versions menu not found");
+        return;
+    }
+
+    // Remove the Reset option and divider
+    const resetItem = menu.querySelector('li:first-child');
+    const divider = menu.querySelector('.divider');
+    if (resetItem) resetItem.remove();
+    if (divider) divider.remove();
+
+    for (const [key, value] of Object.entries(versions)) {
+        const link = Array.from(menu.getElementsByTagName('a'))
+            .find(a => a.textContent.trim() === key);
+
+        assert(link);
+        link.href = value;
+    }
+}
+
+// Start observing the navbar for changes
+document.addEventListener('DOMContentLoaded', () => {
+    const navbar = document.querySelector('#navbar-wrapper');
+    if (navbar) {
+        language_observer.observe(navbar, {
+            childList: true,
+            subtree: true
+        });
+    }
+
+    let versions = JSON.parse(document.getElementById('hotdoc-rust-info')
+        .getAttribute("hotdoc-rustdoc-versions")
+        .replace(/'/g, '"'));
+
+
+    createTagsDropdown({ "API versions": Object.keys(versions) });
+
+    transformVersionMenu(versions);
+});
+
+function syncSidenavIframeParams() {
+    const params = new URLSearchParams(window.location.search);
+    const language = params.get('gi-language');
+    const frame = document.getElementById('sitenav-frame');
+
+    if (frame) {
+        const frameUrl = new URL(frame.src, window.location.origin);
+        if (language) {
+            frameUrl.searchParams.set('gi-language', language);
+        } else {
+            frameUrl.searchParams.delete('gi-language');
+        }
+        frame.src = frameUrl.toString();
+    }
+}
+
+document.addEventListener('DOMContentLoaded', syncSidenavIframeParams);
+
+
diff --git a/subprojects/gst-docs/scripts/assets/js/sitemap-rs-fixer.js b/subprojects/gst-docs/scripts/assets/js/sitemap-rs-fixer.js
new file mode 100644
index 0000000000..dd0ee6a0f8
--- /dev/null
+++ b/subprojects/gst-docs/scripts/assets/js/sitemap-rs-fixer.js
@@ -0,0 +1,76 @@
+function modifyLibsPanel() {
+    // Check if we should show Rust API
+    const params = new URLSearchParams(window.location.search);
+    var language = params.get('gi-language');
+
+    if (!language) {
+        language = utils.getStoredLanguage();
+    }
+
+    if (language === 'rust') {
+        let apiref = document.querySelector('a[data-nav-ref="gi-extension-GStreamer-libs.html"]');
+        apiref.innerText = "Rust crates";
+        apiref.href = "rust/stable/latest/docs/index.html?gi-language=rust";
+
+        // Add crates to the panel
+        const crates = CRATES_LIST; // This will be replaced by Python
+        const renames = CRATES_RENAMES; // This will be replaced by Python
+
+        const rootDiv = document.querySelector('div[data-nav-ref="gi-extension-GStreamer-libs.html"]');
+        if (!rootDiv) {
+            console.error('Root div not found');
+            return;
+        }
+        // Now iterate first level panel bodies to make links point to rust
+        // crates
+        const firstLevelPanels = rootDiv.querySelectorAll(':scope > div.sidenav-panel-body');
+        firstLevelPanels.forEach((panel, index) => {
+            if (index >= crates.length) {
+                panel.remove();
+                return;
+            }
+
+            const crate = crates[index];
+            const link = panel.querySelector('a');
+
+            if (link) {
+                // Update href
+                link.setAttribute('href', `rust/stable/latest/docs/${crate}/index.html?gi-language=rust`);
+
+                // Update text content
+                link.textContent = renames[crate] ||
+                    crate.replace("gstreamer", "")
+                        .replace(/_/g, " ")
+                        .trim()
+                        .split(" ")
+                        .map(word => word.charAt(0).toUpperCase() + word.slice(1))
+                        .join(" ");
+
+                // Remove children of the library item
+                const navRef = link.getAttribute('data-nav-ref');
+                if (navRef) {
+                    const siblingDiv = rootDiv.querySelector(`div[data-nav-ref="${navRef}"]`);
+                    if (siblingDiv) {
+                        siblingDiv.remove();
+                    }
+                }
+
+                // And now remove the glyphicons (arrows) as we removed the
+                // children
+                const linkContainer = link.closest('div');
+                if (linkContainer) {
+                    const glyphicons = linkContainer.querySelectorAll('.glyphicon');
+                    glyphicons.forEach(icon => icon.remove());
+                }
+
+            }
+        });
+    }
+}
+
+// Run when page loads and when URL changes
+document.addEventListener('DOMContentLoaded', modifyLibsPanel);
+window.addEventListener('popstate', modifyLibsPanel);
+window.addEventListener('pushstate', modifyLibsPanel);
+window.addEventListener('replacestate', modifyLibsPanel);
+
diff --git a/subprojects/gst-docs/scripts/assets/js/theme-sync.js b/subprojects/gst-docs/scripts/assets/js/theme-sync.js
new file mode 100644
index 0000000000..c9fe57ff1a
--- /dev/null
+++ b/subprojects/gst-docs/scripts/assets/js/theme-sync.js
@@ -0,0 +1,34 @@
+function syncThemeWithHotdoc() {
+    // Get the current stylesheet
+    const currentStyle = document.querySelector('link[rel="stylesheet"][href*="frontend"]');
+    if (!currentStyle) return;
+
+    // Check if we're using dark theme in hotdoc
+    const isDark = getActiveStyleSheet() == 'dark';
+
+    // Use rustdoc's switchTheme function to set the theme
+    let newThemeName = isDark ? 'dark' : 'light';
+    window.switchTheme(newThemeName, true);
+}
+
+// Run on page load
+document.addEventListener('DOMContentLoaded', () => {
+    localStorage.setItem("rustdoc-use-system-theme", false);
+    syncThemeWithHotdoc();
+
+    // Watch for theme changes in hotdoc
+    const theme_observer = new MutationObserver((mutations) => {
+        mutations.forEach((mutation) => {
+            if (mutation.type === 'attributes' && mutation.attributeName === 'disabled') {
+                syncThemeWithHotdoc();
+            }
+        });
+    });
+
+    // Start observing theme changes
+    const styleLink = document.querySelector('link[rel="stylesheet"][href*="frontend"]');
+    if (styleLink) {
+        theme_observer.observe(styleLink, { attributes: true });
+    }
+});
+
diff --git a/subprojects/gst-docs/scripts/rust_doc_unifier.py b/subprojects/gst-docs/scripts/rust_doc_unifier.py
new file mode 100755
index 0000000000..8d884146b4
--- /dev/null
+++ b/subprojects/gst-docs/scripts/rust_doc_unifier.py
@@ -0,0 +1,564 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import json
+import html
+import shutil
+from collections import OrderedDict
+from pathlib import Path
+from bs4 import BeautifulSoup
+import zipfile
+from urllib.request import urlretrieve
+from multiprocessing import Pool, cpu_count, Manager
+from functools import partial
+import traceback
+import gitlab
+
+
+def get_documentation_artifact_url(project_name='gstreamer/gstreamer',
+                                   job_name='documentation',
+                                   branch='main') -> str:
+    """
+    Returns the URL of the latest artifact from GitLab for the specified job.
+
+    Args:
+        project_name (str): Name of the GitLab project
+        job_name (str): Name of the job
+        branch (str): Name of the git branch
+    """
+    gl = gitlab.Gitlab("https://gitlab.freedesktop.org/")
+    project = gl.projects.get(project_name)
+    pipelines = project.pipelines.list(get_all=False)
+    for pipeline in pipelines:
+        if pipeline.ref != branch:
+            continue
+
+        job, = [j for j in pipeline.jobs.list(iterator=True)
+            if j.name == job_name]
+        if job.status != "success":
+            continue
+
+        return f"https://gitlab.freedesktop.org/{project_name}/-/jobs/{job.id}/artifacts/download"
+
+    raise Exception("Could not find documentation artifact")
+
+
+def get_relative_prefix(file_path, docs_root):
+    """
+    Returns the relative path prefix for a given HTML file.
+
+    Args:
+        file_path (Path): Path to the HTML file
+        docs_root (Path): Root directory of the documentation
+    """
+    rel_path = os.path.relpath(docs_root, file_path.parent)
+    if rel_path == '.':
+        return './'
+    return '../' + '../' * rel_path.count(os.sep)
+
+
+def fix_relative_urls(element, prefix):
+    """
+    Fixes relative URLs in a hotdoc component to include the correct prefix.
+
+    Args:
+        element: BeautifulSoup element containing hotdoc navigation or resources
+        prefix: Prefix to add to relative URLs
+    """
+    # Fix href attributes
+    for tag in element.find_all(True, {'href': True}):
+        url = tag['href']
+        if url.startswith(('http://', 'https://', 'mailto:', '#', 'javascript:')):
+            continue
+
+        if url.endswith('/') or '.' not in url.split('/')[-1]:
+            if not url.endswith('index.html'):
+                url = url.rstrip('/') + '/index.html'
+
+        if ".html" in url and '?gi-language=' not in url:
+            url += '?gi-language=rust'
+
+        tag['href'] = prefix + url
+
+    # Fix src attributes
+    for tag in element.find_all(True, {'src': True}):
+        url = tag['src']
+        if not url.startswith(('http://', 'https://', 'data:', 'javascript:')):
+            if '?gi-language=' not in url:
+                url += '?gi-language=rust'
+            tag['src'] = prefix + url
+
+
+def extract_hotdoc_resources(index_html_soup, prefix):
+    """
+    Extracts required CSS and JS resources from the main hotdoc page.
+    Returns tuple of (css_links, js_scripts)
+    """
+    head = index_html_soup.find('head')
+
+    # Extract CSS links
+    css_links = [link for link in head.find_all('link') if 'enable_search.css'
+                 not in link['href']]
+
+    # Extract JS scripts
+    js_scripts = []
+    for script in head.find_all('script'):
+        src = script.get('src', '')
+        if [unwanted for unwanted in ["trie_index.js", "prism-console-min.js", 'trie.js', 'language-menu.js'] if unwanted in src]:
+            continue
+
+        if 'language_switching.js' in script['src']:
+            js_scripts.append(BeautifulSoup('<script src="assets/js/utils.js" />', 'html.parser'))
+
+            # Inject necessary data for the 'language_switching.js' script to
+            # properly populate the Language dropdown menu for us.
+            js_scripts.append(BeautifulSoup(f'''
+<script>
+    utils.hd_context.project_url_path = "/../{prefix}libs.html";
+    utils.hd_context.gi_languages = ['c', 'python', 'javascript', 'rust'];
+</script>
+''', 'html.parser'))
+
+        js_scripts.append(script)
+
+    return css_links, js_scripts
+
+
+def extract_hotdoc_nav(index_html_soup):
+    """
+    Extracts the navigation bar from the main GStreamer page.
+    Returns the navigation HTML.
+    """
+    nav = index_html_soup.find('nav', class_='navbar')
+
+    for tag in nav.find_all(True, {'href': True}):
+        url = tag['href']
+        if "gstreamer/gi-index.html" in url:
+            tag['href'] = "rust/stable/latest/docs/gstreamer/index.html"
+        elif "libs.html" in url:
+            tag['href'] = "rust/stable/latest/docs/index.html"
+
+    return nav
+
+
+def get_hotdoc_components(docs_root, prefix):
+    """
+    Reads the main GStreamer page and extracts required components.
+
+    Returns tuple of (resources_html, nav_html)
+    """
+    index_path = docs_root / "index.html"
+    with open(index_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+
+    soup = BeautifulSoup(content, 'html.parser')
+
+    # Extract resources and navigation first
+    css_links, js_scripts = extract_hotdoc_resources(soup, prefix)
+    nav = extract_hotdoc_nav(soup)
+    if not css_links:
+        raise Exception("Failed to extract CSS links")
+    if not js_scripts:
+        raise Exception("Failed to extract JS scripts")
+    if not nav:
+        raise Exception("Failed to extract navigation")
+
+    resources_soup = BeautifulSoup("<div></div>", 'html.parser')
+    assert resources_soup.div
+    for component in css_links + js_scripts:
+        resources_soup.div.append(component)
+
+    # Fix URLs in the extracted components
+    fix_relative_urls(resources_soup, prefix)
+    fix_relative_urls(nav, prefix)
+
+    # Build final HTML
+    resources_html = "\n".join(str(tag) for tag in resources_soup.div.contents)
+    resources_html += f'\n<script src="{prefix}assets/rustdoc/js/theme-sync.js" />'
+    nav_html = str(nav) if nav else ""
+
+    return resources_html, nav_html
+
+
+def modify_rustdoc_html_file(file_path, docs_root):
+    """Modifies a single HTML file to include hotdoc navigation."""
+    with open(file_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+
+    # Calculate the relative path prefix for this file
+    prefix = get_relative_prefix(file_path, docs_root)
+
+    # Get hotdoc components with fixed URLs
+    resources_html, nav_html = get_hotdoc_components(docs_root, prefix)
+
+    soup = BeautifulSoup(content, 'html.parser')
+
+    remove_rustdoc_settings(soup)
+    if not add_rust_api_menu(soup, prefix):
+        raise Exception("Failed to add Rust API menu")
+
+    # Add hotdoc resources to head
+    head = soup.find('head')
+    if not head or not resources_html:
+        raise Exception("Failed to add get hotdoc components")
+
+    rust_versions = OrderedDict({
+        "Stable": f'{prefix}rust/stable/latest/docs/index.html',
+        "Development": f'{prefix}rust/git/docs/index.html',
+    })
+
+    for file in (docs_root / "rust" / "stable").iterdir():
+        if file.name == "latest":
+            continue
+
+        if file.is_dir() and (file / "index.html").exists():
+            rust_versions[file.name] = f'{prefix}rust/stable/{file.name}/docs/index.html'
+
+    head.insert(0, BeautifulSoup(
+        f'''<meta id="hotdoc-rust-info"
+                hotdoc-root-prefix="{prefix}"
+                hotdoc-rustdoc-versions="{html.escape(json.dumps(rust_versions))}" />''', 'html.parser'))
+    resources = BeautifulSoup(resources_html, 'html.parser')
+    styles = BeautifulSoup(f"<link rel=\"stylesheet\" href=\"{prefix}assets/rustdoc/css/rustdoc-in-hotdoc.css\" />", 'html.parser')
+    head.append(resources)
+    head.append(styles)
+
+    # Add hotdoc navigation
+    body = soup.find('body')
+    if body and nav_html:
+        nav = BeautifulSoup(nav_html, 'html.parser')
+        first_child = body.find(True)
+        if first_child:
+            first_child.insert_before(nav)
+
+    # Write modified content back to file
+    with open(file_path, 'w', encoding='utf-8') as f:
+        f.write(str(soup))
+
+    return True
+
+
+def remove_rustdoc_settings(soup):
+    """Removes the entire rustdoc toolbar."""
+    toolbar = soup.find('rustdoc-toolbar')
+    if toolbar:
+        toolbar.decompose()
+
+
+def copy_rustdoc_integration_assets(docs_dir):
+    """Ensures the rustdoc assets directory exists."""
+    asset_dir = docs_dir / "assets" / "rustdoc"
+    asset_dir.mkdir(parents=True, exist_ok=True)
+
+    src_asset_dir = Path(__file__).parent / "assets"
+    shutil.copytree(src_asset_dir, asset_dir, dirs_exist_ok=True)
+
+    # Get list of non-sys Rust crates
+    latest_file = docs_dir / "rust" / "stable" / "latest"
+    try:
+        with open(latest_file, 'r') as f:
+            version = f.read().strip()
+    except IsADirectoryError:
+        version = 'latest'
+
+    rust_docs_path = docs_dir / "rust" / "stable" / version / "docs"
+    if not rust_docs_path.exists():
+        print(f"Warning: Rust docs directory not found at {rust_docs_path}")
+        return
+
+    crates = []
+    for item in rust_docs_path.iterdir():
+        if item.is_dir() and not item.name.endswith('_sys') and (item / "index.html").exists():
+            crates.append(item.name)
+
+    crates.sort()  # Sort alphabetically
+    crates_renames = {
+        "gstreamer": "core",
+        "gstreamer_gl": "OpenGL",
+        "gstreamer_gl_egl": "OpenGL EGL",
+        "gstreamer_gl_wayland": "OpenGL Wayland",
+        "gstreamer_gl_x11": "OpenGL X11",
+    }
+
+    rs_fixer_script = asset_dir / "js/sitemap-rs-fixer.js"
+    with rs_fixer_script.open("r") as f:
+        script_template = f.read()
+
+    # Replace the placeholder values
+    script_content = script_template.replace(
+        "CRATES_LIST", str(crates)
+    ).replace(
+        "CRATES_RENAMES", str(crates_renames)
+    )
+
+    # Write the modified script
+    print('\nAdding crates information into sitemap-rs-fixer.js')
+    with rs_fixer_script.open("w") as f:
+        f.write(script_content)
+
+    return asset_dir
+
+
+def add_rust_api_menu(soup, prefix=''):
+    """
+    Adds a script to dynamically insert Rust into the language dropdown menu.
+
+    Args:
+        soup: BeautifulSoup object of the HTML content
+
+    Returns:
+        bool: True if modification was needed and successful, False otherwise
+    """
+    # Check if script already exists
+    existing_scripts = soup.find_all('script')
+    to_remove = []
+    for script in existing_scripts:
+        if 'language-menu.js' in script.get('src', ''):
+            to_remove.append(script)
+            break
+    for script in to_remove:
+        existing_scripts.remove(script)
+
+    # Add script to the end of head
+    head = soup.find('head')
+    if not head:
+        raise Exception("Failed to find <head> tag")
+    script_tag = BeautifulSoup(f'<script src="{prefix}assets/rustdoc/js/language-menu.js" />', 'html.parser')
+
+    head.append(script_tag)
+    return True
+
+
+def modify_hotdoc_html_file(file_path):
+    """Modifies a single hotdoc HTML file to add the Rust API menu."""
+    with open(file_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+
+    soup = BeautifulSoup(content, 'html.parser')
+
+    # Add Rust API menu
+    success = add_rust_api_menu(soup)
+    if not success:
+        raise Exception("Failed to add Rust API menu")
+
+    # Write modified content back to file
+    with open(file_path, 'w', encoding='utf-8') as f:
+        f.write(str(soup))
+    return True
+
+
+def is_rustdoc_file(path):
+    # Convert to Path object and get parts
+    parts = Path(path).parts
+
+    # Check if path has any parts and if first part is "rust"
+    return len(parts) > 0 and parts[0] == "rust"
+
+
+def process_single_file(file_path, docs_path, counters):
+    """
+    Process a single HTML file with appropriate modifications.
+
+    Args:
+        file_path (Path): Path to the HTML file
+        docs_path (Path): Root documentation directory
+        counters (dict): Shared dictionary for counting successes/failures
+    """
+    try:
+        if not is_rustdoc_file(file_path.relative_to(docs_path)):
+            if modify_hotdoc_html_file(file_path):
+                with counters['lock']:
+                    counters['processed'] += 1
+            else:
+                with counters['lock']:
+                    counters['failed'] += 1
+            return
+
+        if modify_rustdoc_html_file(file_path, docs_path):
+            with counters['lock']:
+                counters['processed'] += 1
+        else:
+            with counters['lock']:
+                counters['failed'] += 1
+
+        if sys.stdout.isatty():
+            print(f"\rProcessed: {counters['processed'] + counters['failed']}/{counters['total']} files", end='')
+
+    except Exception as e:
+        print(f"\nError processing {file_path}: {repr(e)}")
+        traceback.print_exc()
+        with counters['lock']:
+            counters['failed'] += 1
+
+
+def has_ancestor(path: Path, ancestor: Path) -> bool:
+    try:
+        path.resolve().relative_to(ancestor.resolve())
+        return True
+    except ValueError:
+        return False
+
+
+def process_docs(docs_dir: Path):
+    """
+    Recursively processes all HTML files in the Rust documentation directory using parallel processing.
+
+    Args:
+        docs_dir (str): Path to the root of the Rust documentation
+    """
+    docs_path = Path(docs_dir).resolve()
+
+    if not docs_path.exists():
+        print(f"Error: Directory {docs_dir} does not exist")
+        return
+
+    sitemap_path = (docs_path / "hotdoc-sitemap.html").resolve()
+    assets_dir = (docs_path / 'assets').resolve()
+
+    def html_file_needs_processing(html_file):
+        html_file = html_file.resolve()
+        if has_ancestor(html_file, assets_dir):
+            return False
+        if html_file == sitemap_path:
+            return False
+        return True
+
+    # Get list of all HTML files
+    html_files = [html_file for html_file in docs_path.rglob('*.html') if
+                  html_file_needs_processing(html_file)]
+
+    # Create a manager for sharing counters between processes
+    manager = Manager()
+    counters = manager.dict({
+        'processed': 0,
+        'failed': 0,
+        'total': len(html_files)
+    })
+    counters['lock'] = manager.Lock()
+
+    # Calculate optimal number of processes
+    num_processes = min(cpu_count(), len(html_files))
+
+    print(f"\nStarting parallel processing with {num_processes} processes...")
+
+    # Create a partial function with fixed arguments
+    process_func = partial(process_single_file, docs_path=docs_path, counters=counters)
+
+    # process_single_file(Path('/Users/thiblahute/fs-devel/gstreamer/tmptestdoc/documentation/rust/stable/0.23/docs/gstreamer/index.html'), docs_path, counters)
+    # for html_file in html_files:
+    #     process_func(html_file)
+    # Process files in parallel
+    with Pool(processes=num_processes) as pool:
+        pool.map(process_func, html_files)
+
+    print("\n\nProcessing complete:")
+    print(f"Successfully processed: {counters['processed']} files")
+    print(f"Failed to process: {counters['failed']} files")
+
+
+def download_rust_docs(doc_dir):
+    """
+    Downloads the latest Rust documentation from rust-lang.org and unzips it.
+
+    Args:
+        doc_dir (str): Path to the directory where the documentation will be downloaded
+    """
+    print("Looking for rust latest rust documention")
+    if not os.path.exists("rustdocs.zip"):
+        rustdoc_url = get_documentation_artifact_url("gstreamer/gstreamer-rs",
+                                                     "pages")
+
+        def progress_hook(count, block_size, total_size):
+            percent = int(count * block_size * 100 / total_size)
+            sys.stdout.write(f"\rDownloading... {percent}%")
+            sys.stdout.flush()
+
+        print(f"Downloading rust documentation from {rustdoc_url}")
+        urlretrieve(rustdoc_url, "rustdocs.zip", reporthook=progress_hook)
+
+    print("Unpacking rust documentation")
+    with zipfile.ZipFile('rustdocs.zip', 'r') as zip_ref:
+        # Get list of files
+        file_list = zip_ref.namelist()
+        total_files = len(file_list)
+
+        # Extract each file with progress
+        for i, file in enumerate(file_list, 1):
+            if file.endswith('html.gz'):
+                continue
+
+            zip_ref.extract(file, '.')
+            if sys.stdout.isatty():
+                print(f"\rExtracting: {i}/{total_files} files", end='')
+
+    shutil.move("public", doc_dir / "rust")
+
+
+def move_rust_latest(doc_dir):
+    # Handle latest symlink
+    latest_file = doc_dir / "rust" / "stable" / "latest"
+    assert latest_file.exists
+    if latest_file.is_dir():
+        print(f'{latest_file} is already a directory')
+        return
+
+    # Read the version from the latest file
+    with open(latest_file, 'r') as f:
+        version = f.read().strip()
+
+    # Remove the 'latest' file
+    latest_file.unlink()
+
+    # Create symlink from version directory to 'latest'
+    version_dir = latest_file.parent / version
+    assert version_dir.exists(), f"Version directory '{version}' not found"
+    version_dir.rename(latest_file)
+    print(f'Moved {version_dir} to {latest_file}')
+
+
+def add_rust_fixer_to_sitemap(docs_root: Path):
+    """Modifies the hotdoc-sitemap.html file to customize the Rust API references."""
+    sitemap_path = docs_root / "hotdoc-sitemap.html"
+    if not sitemap_path.exists():
+        print(f"Warning: {sitemap_path} not found")
+        return
+
+    # Read the existing sitemap to extract the structure
+    with open(sitemap_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+
+    soup = BeautifulSoup(content, 'html.parser')
+
+    head = soup.find('head')
+    assert head, "Failed to find <head> tag"
+    for script in head.find_all('script'):
+        src = script.get('src', '')
+        if 'sitemap-rs-fixer.js' in src:
+            print('hotdoc-sitemap.html already contains the script')
+            return
+
+    # The utils script is required byt the fixer to detect
+    # the configured language
+    head.append(BeautifulSoup('''
+    <script src="assets/js/utils.js" />
+    <script src="assets/rustdoc/js/sitemap-rs-fixer.js" />
+                ''', 'html.parser'))
+
+    with open(sitemap_path, 'w', encoding='utf-8') as f:
+        f.write(str(soup))
+
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) != 2:
+        print("Usage: python modify_rust_docs.py <rust_docs_directory>")
+        sys.exit(1)
+
+    docs_dir = Path(sys.argv[1])
+    download_rust_docs(docs_dir)
+    move_rust_latest(docs_dir)
+    copy_rustdoc_integration_assets(docs_dir)
+    process_docs(docs_dir)
+    add_rust_fixer_to_sitemap(docs_dir)