From 8b143a1ca87433c84855a801f406b198deab61df Mon Sep 17 00:00:00 2001 From: KaySar12 Date: Thu, 20 Mar 2025 14:36:46 +0700 Subject: [PATCH] update --- .../hr_timesheet/security/ir.model.access.csv | 1 + setup/color_log.py | 2 +- setup/delete_records.py | 250 ----------------- setup/record_cleaner.py | 259 ++++++++++++++++++ 4 files changed, 261 insertions(+), 251 deletions(-) delete mode 100755 setup/delete_records.py create mode 100755 setup/record_cleaner.py diff --git a/addons/hr_timesheet/security/ir.model.access.csv b/addons/hr_timesheet/security/ir.model.access.csv index ecc3e45ed..e45fc8ac3 100644 --- a/addons/hr_timesheet/security/ir.model.access.csv +++ b/addons/hr_timesheet/security/ir.model.access.csv @@ -5,3 +5,4 @@ access_uom_uom_hr_timesheet,uom.uom.timesheet.user,uom.model_uom_uom,hr_timeshee access_project_project,project.project.timesheet.user,model_project_project,hr_timesheet.group_hr_timesheet_user,1,0,0,0 access_timesheets_analysis_report,timesheets.analysis.report,model_timesheets_analysis_report,base.group_user,1,0,0,0 access_hr_employee_delete_wizard,hr.employee.delete.wizard,model_hr_employee_delete_wizard,hr.group_hr_user,1,1,1,0 +s \ No newline at end of file diff --git a/setup/color_log.py b/setup/color_log.py index 7a31f7906..58d51a14e 100644 --- a/setup/color_log.py +++ b/setup/color_log.py @@ -19,7 +19,7 @@ def Show(status, message): colorize("[", "90") + colorize(" INFO ", "38;5;154") + colorize("]", "90") ), # Green, Grey 3: ( - colorize("[", "90") + colorize(" NOTICE ", "33") + colorize("]", "90") + colorize("[", "90") + colorize(" WARNING ", "33") + colorize("]", "90") ), # Yellow, Grey } print(f"{colors.get(status, '')} {message}") diff --git a/setup/delete_records.py b/setup/delete_records.py deleted file mode 100755 index a694d5fea..000000000 --- a/setup/delete_records.py +++ /dev/null @@ -1,250 +0,0 @@ -#!/usr/bin/env python -""" -Delete records from an Odoo database based on a model and domain filter. - -Usage: - delete_records.py - -Example: - delete_records.py mydb res.partner --domain "[('active', '=', False)]" --force -""" -import argparse -import ast -import json -import multiprocessing as mp -import os -import odoorpc -import color_log - -# Default configuration -DEFAULT_HOST = "localhost" -DEFAULT_PORT = 8069 -DEFAULT_USERNAME = "admin" -DEFAULT_PASSWORD = "admin" -DEFAULT_DOMAIN = "[]" -DEFAULT_PROCESS_SIZE = 30 - -# Logging levels -OK, FAIL, INFO, NOTICE = 0, 1, 2, 3 - - -def parse_arguments(): - """Parse command-line arguments.""" - parser = argparse.ArgumentParser(description="Delete records from an Odoo model.") - parser.add_argument("db_name", help="Database name") - parser.add_argument("base_model", help="Model to delete records from") - parser.add_argument("--host", default=DEFAULT_HOST, help="Odoo server host") - parser.add_argument( - "--port", type=int, default=DEFAULT_PORT, help="Odoo server port" - ) - parser.add_argument("--username", default=DEFAULT_USERNAME, help="Odoo username") - parser.add_argument("--password", default=DEFAULT_PASSWORD, help="Odoo password") - parser.add_argument("--domain", default=DEFAULT_DOMAIN, help="Domain filter") - parser.add_argument( - "--process_size", - type=int, - default=DEFAULT_PROCESS_SIZE, - help="Number of parallel processes", - ) - parser.add_argument( - "--force", action="store_true", help="Force delete instead of archiving" - ) - parser.add_argument( - "--soft", action="store_true", help="Archive instead of deleting" - ) - parser.add_argument( - "--refresh-cache", action="store_true", help="Refresh related models cache" - ) - return parser.parse_args() - - -def connect_to_odoo(host, port, db_name, username, password): - """Establish connection to Odoo database.""" - odoo = odoorpc.ODOO(host, port=port) - color_log.Show(INFO, f"Available databases: {odoo.db.list()}") - try: - odoo.login(db_name, username, password) - color_log.Show(OK, f"Connected to {host}:{port}, DB: {db_name}") - return odoo - except odoorpc.error.RPCError as e: - color_log.Show(FAIL, f"Login failed: {e}") - exit(1) - except Exception as e: - color_log.Show(FAIL, f"Connection error: {e}") - exit(1) - - -def parse_domain(domain_str): - """Convert domain string to a list and validate it.""" - try: - domain = ast.literal_eval(domain_str) - if not isinstance(domain, list) or not all( - isinstance(t, (tuple, list)) and len(t) == 3 for t in domain - ): - raise ValueError("Domain must be a list of 3-tuples") - return domain - except (ValueError, SyntaxError) as e: - color_log.Show(FAIL, f"Invalid domain format: {e}") - exit(1) - - -def get_related_fields(odoo, db_name, base_model, process_size, refresh_cache=False): - """Fetch related fields for the base model, using cache if available.""" - cache_file = f"cache/{db_name}/{base_model}.cache.json" - if not refresh_cache and os.path.exists(cache_file): - with open(cache_file, "r") as f: - color_log.Show(INFO, f"Loaded related models from cache: {base_model}") - return json.load(f) - - color_log.Show(INFO, f"Fetching related models for {base_model}...") - related_models = {} - skipped_models = set() - - def fetch_fields(model_name): - if model_name in skipped_models: - return {} - try: - fields = odoo.env[model_name].fields_get() - related = {} - for name, info in fields.items(): - if ( - info.get("type") in ["many2one", "many2many", "one2many"] - and info.get("relation") == base_model - ): - related.setdefault(model_name, []).append(name) - return related - except Exception as e: - color_log.Show(NOTICE, f"Skipping {model_name}: {e}") - skipped_models.add(model_name) - return {} - - model_ids = odoo.env["ir.model"].search([]) - models = [m["model"] for m in odoo.env["ir.model"].read(model_ids, ["model"])] - with mp.Pool(processes=process_size) as pool: - results = pool.map(fetch_fields, models) - for result in results: - related_models.update(result) - - os.makedirs(os.path.dirname(cache_file), exist_ok=True) - with open(cache_file, "w") as f: - json.dump(related_models, f, indent=4) - return related_models - - -def unreference_record(odoo, record_id, model_name, related_models, process_id): - """Remove references to a record from related models.""" - for related_model, fields in related_models.items(): - related_obj = odoo.env[related_model] - for field in fields: - try: - related_ids = related_obj.search([(field, "=", record_id)]) - if related_ids: - related_obj.browse(related_ids).write({field: False}) - color_log.Show( - OK, - f"{process_id}: Unreferenced {record_id} from {related_model} ({field})", - ) - except Exception as e: - color_log.Show( - NOTICE, - f"{process_id}: Error unreferencing {record_id} in {related_model}: {e}", - ) - - -def delete_batch(odoo, batch, model_name, related_models, process_id, force, soft): - """Delete or archive a batch of records.""" - model = odoo.env[model_name] - deleted, archived, skipped = 0, 0, 0 - - for record_id in batch: - is_referenced = any( - odoo.env[m].search_count([(f, "=", record_id)]) > 0 - for m, fields in related_models.items() - for f in fields - ) - try: - if is_referenced and not force: - model.browse(record_id).write({"active": False}) - archived += 1 - color_log.Show( - INFO, f"{process_id}: Archived {model_name} ID {record_id}" - ) - elif not soft or force: - model.browse(record_id).unlink() - deleted += 1 - color_log.Show( - INFO, f"{process_id}: Deleted {model_name} ID {record_id}" - ) - except Exception as e: - if force: - try: - unreference_record( - odoo, record_id, model_name, related_models, process_id - ) - model.browse(record_id).unlink() - deleted += 1 - color_log.Show( - INFO, f"{process_id}: Force deleted {model_name} ID {record_id}" - ) - except Exception as e: - skipped += 1 - color_log.Show(NOTICE, f"{process_id}: Skipped {record_id}: {e}") - else: - skipped += 1 - color_log.Show(INFO, f"{process_id}: Skipped {record_id}: {e}") - - color_log.Show( - OK, - f"{process_id}: {model_name} - Deleted: {deleted}, Archived: {archived}, Skipped: {skipped}", - ) - - -def main(): - """Orchestrate the deletion process.""" - args = parse_arguments() - odoo = connect_to_odoo( - args.host, args.port, args.db_name, args.username, args.password - ) - domain = parse_domain(args.domain) - - model = odoo.env[args.base_model] - record_ids = model.search(domain) - if not record_ids: - color_log.Show( - FAIL, f"No records found in {args.base_model} with domain {args.domain}" - ) - return - - related_models = get_related_fields( - odoo, args.db_name, args.base_model, args.process_size, args.refresh_cache - ) - batches = [ - record_ids[i : i + args.process_size] - for i in range(0, len(record_ids), args.process_size) - ] - - processes = [] - for i, batch in enumerate(batches, 1): - p = mp.Process( - target=delete_batch, - args=( - odoo, - batch, - args.base_model, - related_models, - f"Process-{i}", - args.force, - args.soft, - ), - ) - processes.append(p) - p.start() - - for p in processes: - p.join() - - color_log.Show(OK, "Deletion process completed.") - - -if __name__ == "__main__": - main() diff --git a/setup/record_cleaner.py b/setup/record_cleaner.py new file mode 100755 index 000000000..0f97cdb23 --- /dev/null +++ b/setup/record_cleaner.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python +""" +Delete records from an Odoo database based on a model and domain filter. + +Usage: + delete_records.py + +Example: + delete_records.py mydb res.partner --domain "[('active', '=', False)]" --force +""" +import argparse +import ast +import json +import multiprocessing as mp +import os +import sys +from typing import Dict, List, Tuple +from functools import partial +import odoorpc +import color_log + +# Default configuration +DEFAULT_HOST = "localhost" +DEFAULT_PORT = 8069 +DEFAULT_USERNAME = "admin" +DEFAULT_PASSWORD = "admin" +DEFAULT_DOMAIN = "[]" +DEFAULT_PROCESS_SIZE = min(mp.cpu_count() * 2, 32) # Dynamic default based on CPU +CACHE_DIR = "cache" +CHUNK_SIZE = 500 # Records per batch for search operations +# Logging levels +OK, FAIL, INFO, WARNING = 0, 1, 2, 3 + + +def parse_arguments() -> argparse.Namespace: + """Parse and validate command-line arguments.""" + parser = argparse.ArgumentParser( + description="Safely delete records from an Odoo model with referential integrity checks.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("db_name", help="Database name") + parser.add_argument("base_model", help="Model to delete records from") + parser.add_argument("--host", default=DEFAULT_HOST, help="Odoo server host") + parser.add_argument( + "--port", type=int, default=DEFAULT_PORT, help="Odoo server port" + ) + parser.add_argument("--username", default=DEFAULT_USERNAME, help="Odoo username") + parser.add_argument("--password", default=DEFAULT_PASSWORD, help="Odoo password") + parser.add_argument( + "--domain", default=DEFAULT_DOMAIN, help="Domain filter as Python list" + ) + parser.add_argument( + "--process-size", + type=int, + default=DEFAULT_PROCESS_SIZE, + help="Number of parallel processes", + ) + parser.add_argument( + "--chunk-size", + type=int, + default=CHUNK_SIZE, + help="Records per batch for search operations", + ) + + action_group = parser.add_mutually_exclusive_group() + action_group.add_argument( + "--force", + action="store_true", + help="Force delete with referential integrity bypass", + ) + parser.add_argument( + "--refresh-cache", action="store_true", help="Refresh related models cache" + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Simulate operations without making changes", + ) + parser.add_argument("--verbose", action="store_true", help="Show detailed output") + + args = parser.parse_args() + + # Validate domain syntax early + try: + ast.literal_eval(args.domain) + except (ValueError, SyntaxError) as e: + color_log.Show(FAIL, f"Invalid domain syntax: {e}") + sys.exit(1) + + return args + + +def connect_to_odoo(args: argparse.Namespace) -> odoorpc.ODOO: + """Establish and verify Odoo connection.""" + try: + odoo = odoorpc.ODOO(args.host, port=args.port) + if args.verbose: + color_log.Show(INFO, f"Available databases: {odoo.db.list()}") + + odoo.login(args.db_name, args.username, args.password) + color_log.Show(OK, f"Connected to {args.host}:{args.port}, DB: {args.db_name}") + return odoo + except odoorpc.error.RPCError as e: + color_log.Show(FAIL, f"Login failed: {e}") + sys.exit(1) + except Exception as e: + color_log.Show(FAIL, f"Connection error: {e}") + sys.exit(1) + + +def get_related_fields( + odoo: odoorpc.ODOO, args: argparse.Namespace +) -> Dict[str, List[str]]: + """Retrieve related fields with cache management.""" + cache_path = os.path.join(CACHE_DIR, args.db_name, f"{args.base_model}.json") + os.makedirs(os.path.dirname(cache_path), exist_ok=True) + + if not args.refresh_cache and os.path.exists(cache_path): + with open(cache_path, "r") as f: + color_log.Show(INFO, f"Loaded related models from cache: {args.base_model}") + return json.load(f) + + color_log.Show(INFO, f"Building related models cache for {args.base_model}...") + related = {} + Model = odoo.env["ir.model"] + model_ids = Model.search([("model", "!=", args.base_model)]) + + for model in Model.read(model_ids, ["model"]): + try: + fields = odoo.env[model["model"]].fields_get() + related_fields = [ + name + for name, desc in fields.items() + if desc.get("relation") == args.base_model + and desc.get("type") in ["many2one", "many2many", "one2many"] + ] + if related_fields: + related[model["model"]] = related_fields + except Exception as e: + if args.verbose: + color_log.Show(WARNING, f"Skipping {model['model']}: {str(e)}") + + with open(cache_path, "w") as f: + json.dump(related, f, indent=2) + + return related + + +def chunker(seq: List[int], size: int) -> List[List[int]]: + """Efficient batch generator.""" + return [seq[pos : pos + size] for pos in range(0, len(seq), size)] + + +def process_batch( + args: argparse.Namespace, batch: List[int], related: Dict[str, List[str]] +) -> Tuple[int, int, int]: + """Process a batch of records with proper error handling.""" + deleted = archived = skipped = 0 + odoo = connect_to_odoo(args) + model = odoo.env[args.base_model] + + for record_id in batch: + try: + if args.dry_run: + color_log.Show(INFO, f"[DRY-RUN] Would process record {record_id}") + continue + + # Check references + if not args.force: + referenced = any( + odoo.env[rel_model].search_count([(field, "=", record_id)]) + for rel_model, fields in related.items() + for field in fields + ) + if referenced: + model.write([record_id], {"active": False}) + archived += 1 + color_log.Show(OK, f"Archived {args.base_model} ID {record_id}") + continue + + else: + model.unlink([record_id]) + deleted += 1 + color_log.Show(OK, f"Deleted {args.base_model} ID {record_id}") + + except odoorpc.error.RPCError as e: + color_log.Show(WARNING, f"Error processing {record_id}: {e}") + skipped += 1 + except Exception as e: + color_log.Show(WARNING, f"Unexpected error with {record_id}: {e}") + skipped += 1 + + return deleted, archived, skipped + + +def main(): + """Main execution flow.""" + args = parse_arguments() + odoo = connect_to_odoo(args) + + # Validate model exists + if args.base_model not in odoo.env: + color_log.Show(FAIL, f"Model {args.base_model} does not exist") + sys.exit(1) + + # Retrieve records + domain = ast.literal_eval(args.domain) + record_ids = odoo.env[args.base_model].search( + domain, offset=0, limit=None, order="id" + ) + if not record_ids: + color_log.Show( + WARNING, f"No records found in {args.base_model} with domain {domain}" + ) + return + + color_log.Show(INFO, f"Found {len(record_ids)} records to process") + + # Prepare related models data + related = get_related_fields(odoo, args) + if related and args.verbose: + color_log.Show(INFO, f"Related models: {json.dumps(related, indent=2)}") + + # Parallel processing + batches = chunker(record_ids, args.chunk_size) + color_log.Show( + INFO, f"Processing {len(batches)} batches with {args.process_size} workers" + ) + + total_stats = [0, 0, 0] + with mp.Pool(args.process_size) as pool: + results = pool.imap_unordered( + partial(process_batch, args, related=related), batches + ) + + for deleted, archived, skipped in results: + total_stats[0] += deleted + total_stats[1] += archived + total_stats[2] += skipped + + # Final report + color_log.Show(OK, "\nOperation summary:") + color_log.Show(OK, f"Total deleted: {total_stats[0]}") + color_log.Show(OK, f"Total archived: {total_stats[1]}") + color_log.Show(OK, f"Total skipped: {total_stats[2]}") + color_log.Show( + OK, f"Success rate: {(total_stats[0]+total_stats[1])/len(record_ids)*100:.1f}%" + ) + + if args.dry_run: + color_log.Show(WARNING, "Dry-run mode: No changes were made to the database") + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + color_log.Show(FAIL, "\nOperation cancelled by user") + sys.exit(1)