documentation/convert2md.sh

#!/bin/bash
# Enable stricter error handling
set -euo pipefail
# Check arguments
if [[ $# -ne 2 ]]; then
    echo "Usage: $0 <source_folder> <target_folder>" >&2
    exit 1
fi
readonly SOURCE_DIR="$1"
readonly TARGET_DIR="$2"
readonly TEMP_ERROR=$(mktemp)
readonly LOG_FILE="/tmp/convert2md_$$.log"
# Trap to clean up temp files on exit
trap 'rm -f "$TEMP_ERROR" "$LOG_FILE"' EXIT

# Validate inputs
validate_inputs() {
    [[ ! -d "$SOURCE_DIR" ]] && {
        echo "Error: Source directory '$SOURCE_DIR' does not exist." >&2
        exit 1
    }
    mkdir -p "$TARGET_DIR" || {
        echo "Error: Could not create target directory '$TARGET_DIR'." >&2
        exit 1
    }
    command -v rst2myst >/dev/null 2>&1 || {
        echo "Error: rst2myst is not installed. Install with 'pip install rst-to-myst'." >&2
        exit 1
    }
}

# Process RST files
process_rst_file() {
    local rst_file="$1"
    local relative_path="${rst_file#$SOURCE_DIR/}"
    local md_file_name="${relative_path%.rst}.md"
    local target_file="$TARGET_DIR/$md_file_name"
    local target_dir=$(dirname "$target_file")
    mkdir -p "$target_dir" || {
        echo "Error: Could not create directory '$target_dir' for '$rst_file'" >&2
        return 1
    }
    if rst2myst stream "$rst_file" > "$target_file" 2>>"$TEMP_ERROR"; then
        echo "Converted: $rst_file -> $target_file" | tee -a "$LOG_FILE"
        echo "converted" >> "$TEMP_ERROR"
        return 0
    else
        echo "Failed to convert: $rst_file" >&2
        cat "$TEMP_ERROR" >&2
        echo "failed" >> "$TEMP_ERROR"
        return 1
    fi
}

# Copy non-RST files
copy_non_rst_file() {
    local file="$1"
    local relative_path="${file#$SOURCE_DIR/}"
    local target_file="$TARGET_DIR/$relative_path"
    local target_dir=$(dirname "$target_file")
    mkdir -p "$target_dir" && cp -p "$file" "$target_file" 2>/dev/null && {
        echo "Copied: $file -> $target_file" | tee -a "$LOG_FILE"
        return 0
    } || {
        echo "Failed to copy: $file" >&2
        return 1
    }
}

main() {
    validate_inputs
    echo "Starting conversion process..." | tee "$LOG_FILE"
    
    # Count total RST files
    readonly TOTAL_RST_FILES=$(find "$SOURCE_DIR" -type f -name "*.rst" | wc -l)
    echo "Found $TOTAL_RST_FILES RST files to process" | tee -a "$LOG_FILE"
    
    local parallel_jobs=$(( $(nproc) / 2 ))  # Use half the CPU cores to avoid overload
    # Export functions for xargs
    export -f process_rst_file copy_non_rst_file
    export SOURCE_DIR TARGET_DIR TEMP_ERROR LOG_FILE
    
    # Process RST files with limited parallelism
    find "$SOURCE_DIR" -type f -name "*.rst" -print0 | xargs -0 -P "$parallel_jobs" -I {} bash -c 'process_rst_file "{}"' || {
        echo "Warning: Some RST processing failed. Check $LOG_FILE for details." >&2
    }
    
    # Process non-RST files with limited parallelism
    find "$SOURCE_DIR" -type f ! -name "*.rst" -print0 | xargs -0 -P "$parallel_jobs" -I {} bash -c 'copy_non_rst_file "{}"' || {
        echo "Warning: Some file copies failed. Check $LOG_FILE for details." >&2
    }
    
    # Count results from log
    readonly SUCCESSFUL_CONVERSIONS=$(grep -c "^Converted:" "$LOG_FILE")
    readonly FAILED_CONVERSIONS=$(grep -c "^Failed to convert:" "$LOG_FILE")
    
    # Summary
    echo "Conversion and copy process complete." | tee -a "$LOG_FILE"
    echo "Total RST files found: $TOTAL_RST_FILES" | tee -a "$LOG_FILE"
    echo "RST files successfully converted: $SUCCESSFUL_CONVERSIONS" | tee -a "$LOG_FILE"
    echo "RST files failed to convert: $FAILED_CONVERSIONS" | tee -a "$LOG_FILE"
    echo "Non-RST files copied: $(grep -c "^Copied:" "$LOG_FILE")" | tee -a "$LOG_FILE"
    
    # Check if all files were processed
    if [[ $((TOTAL_RST_FILES)) != $((SUCCESSFUL_CONVERSIONS + FAILED_CONVERSIONS)) ]]; then
        echo "Warning: Some RST files might not have been processed!" >&2
        exit 1
    fi
    
    [[ $FAILED_CONVERSIONS -gt 0 ]] && {
        echo "Note: Some RST conversions failed. See $LOG_FILE for details." >&2
        exit 1
    }
}

main "$@"