# credativ-pg-migrator
# Copyright (C) 2025 credativ GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import concurrent.futures
import importlib
from credativ_pg_migrator.migrator_logging import MigratorLogger
from credativ_pg_migrator.migrator_tables import MigratorTables
from credativ_pg_migrator.constants import MigratorConstants
import traceback
import uuid
import fnmatch
import re
import time
import json
import os

class Orchestrator:
    def __init__(self, config_parser):
        self.config_parser = config_parser
        self.logger = MigratorLogger(self.config_parser.get_log_file()).logger
        self.source_connection = self.load_connector('source')
        self.target_connection = self.load_connector('target')
        self.migrator_tables = MigratorTables(self.logger, self.config_parser)
        self.on_error_action = self.config_parser.get_on_error_action()
        self.source_schema_name = self.config_parser.get_source_schema()
        self.target_schema_name = self.config_parser.get_target_schema()
        if self.config_parser.is_resume_after_crash():
            self.migrator_tables.insert_main({'task_name': 'Orchestrator', 'subtask_name': 'Resume after crash'})
            self.config_parser.print_log_message('INFO', "orchestrator: __init__: #############################################################################")
            self.config_parser.print_log_message('INFO', "orchestrator: __init__: Orchestration: Resuming migration after crash - stats from crashed migration:")
            self.config_parser.print_log_message('INFO', f"orchestrator: __init__: Orchestration: drop_unfinished-tables set to: {self.config_parser.should_drop_unfinished_tables()}")
            self.migrator_tables.print_migration_summary()
            self.config_parser.print_log_message('INFO', "orchestrator: __init__: #############################################################################")
            self.config_parser.print_log_message('INFO', "orchestrator: __init__: Orchestration: Continuing migration...")
        else:
            self.migrator_tables.insert_main({'task_name': 'Orchestrator', 'subtask_name': ''})

    def run(self):
        if self.config_parser.is_standard_workflow():
            self.config_parser.print_log_message('INFO', "orchestrator: run: Standard workflow")

            try:
                self.config_parser.print_log_message('INFO', "orchestrator: run: Starting Orchestrator...")

                if self.config_parser.is_resume_after_crash():
                    self.config_parser.print_log_message('INFO', "orchestrator: run: In current version of crash recovery we assume user defined types and domains already exist, so we skip them.")
                else:
                    self.run_create_user_defined_types()
                    self.check_pausing_resuming()

                    ## migration of domains is a bit unclear currently
                    ## domains in PostgreSQL are special data types
                    ## But in Sybase ASE they are defined as sort of additional check constraint on the column
                    self.run_create_domains()

                # In case of crash recovery, we currently continue from this point as in normal migration
                if not self.config_parser.is_dry_run():
                    self.stdwf_migrate_sequences()
                    self.check_pausing_resuming()

                    self.stdwf_migrate_tables()
                    self.check_pausing_resuming()

                    self.stdwf_migrate_indexes('standard')
                    self.check_pausing_resuming()

                    self.stdwf_migrate_constraints()
                    self.check_pausing_resuming()

                    self.stdwf_migrate_views()
                    self.check_pausing_resuming()

                    self.stdwf_migrate_funcprocs()
                    self.check_pausing_resuming()

                    self.stdwf_migrate_triggers()
                    self.check_pausing_resuming()

                    self.stdwf_migrate_indexes('function_based')
                    self.check_pausing_resuming()

                    self.stdwf_migrate_comments()
                    self.check_pausing_resuming()

                    self.run_post_migration_script()
                else:
                    self.config_parser.print_log_message('INFO', "orchestrator: run: Dry run mode enabled. No data migration performed.")

                self.config_parser.print_log_message('INFO', "orchestrator: run: Orchestration complete.")
                self.migrator_tables.update_main_status({'task_name': 'Orchestrator', 'subtask_name': '', 'success': True, 'message': 'finished OK'})

            except Exception as e:
                self.migrator_tables.update_main_status({'task_name': 'Orchestrator', 'subtask_name': '', 'success': False, 'message': f'ERROR: {e}'})
                self.handle_error(e, 'orchestration')

        elif self.config_parser.is_mapping_workflow():
            self.config_parser.print_log_message('INFO', "orchestrator: run: Mapping workflow")

            try:
                if self.config_parser.get_suspend_indexes_constraints():
                    self.mapping_drop_indexes_and_constraints()
                self.mapping_copy_data()
                if self.config_parser.get_suspend_indexes_constraints():
                    self.mapping_create_indexes_and_constraints()
                    self.mapping_check_indexes_and_constraints()
            except Exception as e:
                self.migrator_tables.update_main_status({'task_name': 'Orchestrator', 'subtask_name': '', 'success': False, 'message': f'ERROR: {e}'})
                self.handle_error(e, 'orchestration')

            report_filename = self.config_parser.get_mapping_report_filename()
            if report_filename:
                self.config_parser.print_log_message('INFO', f"orchestrator: run: Generating mapping report to {report_filename}")
                self.migrator_tables.generate_mapping_report(report_filename)

        elif self.config_parser.is_anonymization_workflow():
            self.config_parser.print_log_message('INFO', "orchestrator: run: Anonymization workflow")
            try:
                from credativ_pg_migrator.anonymization.workflow import AnonymizationWorkflow
                workflow = AnonymizationWorkflow(self)
                workflow.run()
            except Exception as e:
                self.migrator_tables.update_main_status({'task_name': 'Orchestrator', 'subtask_name': '', 'success': False, 'message': f'ERROR: {e}'})
                self.handle_error(e, 'orchestration')

        self.migrator_tables.print_migration_summary()

        try:
            self.source_connection.disconnect()
        except Exception as e:
            pass
        try:
            self.target_connection.disconnect()
        except Exception as e:
            pass


    def mapping_drop_indexes_and_constraints(self):
        self.config_parser.print_log_message('INFO', "orchestrator: mapping_drop_indexes_and_constraints: Starting global drop for constraints and indexes")
        target_conn = self.load_connector('target')
        target_conn.connect()

        # 1. Drop constraints
        self.config_parser.print_log_message('INFO', "orchestrator: mapping_drop_indexes_and_constraints: Dropping non-primary key constraints")
        target_constraints = self.migrator_tables.fetch_all_mapping_target_constraints()
        for constraint in target_constraints:
            if constraint['constraint_type'] != 'PRIMARY KEY':
                self.config_parser.print_log_message('DEBUG', f"orchestrator: Dropping constraint {constraint['constraint_name']} on {constraint['target_schema_name']}.{constraint['target_table_name']}")
                drop_sql = f'ALTER TABLE "{constraint["target_schema_name"]}"."{constraint["target_table_name"]}" DROP CONSTRAINT IF EXISTS "{constraint["constraint_name"]}"'
                try:
                    target_conn.execute_query(drop_sql)
                    self.migrator_tables.update_mapping_target_constraint_drop_status({'row_id': constraint['id'], 'dropped': True, 'message': 'dropped OK'})
                except Exception as ex:
                    self.migrator_tables.update_mapping_target_constraint_drop_status({'row_id': constraint['id'], 'dropped': False, 'message': f'ERROR: {ex}'})

        # 2. Drop indexes
        self.config_parser.print_log_message('INFO', "orchestrator: mapping_drop_indexes_and_constraints: Dropping non-primary key indexes")
        target_indexes = self.migrator_tables.fetch_all_mapping_target_indexes()
        constraint_index_info = {c['constraint_name']: c['constraint_type'] for c in target_constraints if c['constraint_type'] in ('UNIQUE', 'EXCLUSION')}

        for index in target_indexes:
            if not index.get('is_primary_key', False):
                if index['index_name'] not in constraint_index_info:
                    self.config_parser.print_log_message('DEBUG', f"orchestrator: Dropping index {index['index_name']} on {index['target_schema_name']}.{index['target_table_name']}")
                    drop_sql = f'DROP INDEX IF EXISTS "{index["target_schema_name"]}"."{index["index_name"]}"'
                    try:
                        target_conn.execute_query(drop_sql)
                        self.migrator_tables.update_mapping_target_index_drop_status({'row_id': index['id'], 'dropped': True, 'message': 'dropped OK'})
                    except Exception as ex:
                        self.migrator_tables.update_mapping_target_index_drop_status({'row_id': index['id'], 'dropped': False, 'message': f'ERROR: {ex}'})
                else:
                    c_type = constraint_index_info[index['index_name']]
                    self.config_parser.print_log_message('DEBUG', f"orchestrator: Skipping index drop {index['index_name']} on {index['target_schema_name']}.{index['target_table_name']} (handled by {c_type} constraint '{index['index_name']}')")

        target_conn.disconnect()

    def mapping_create_indexes_and_constraints(self):
        self.config_parser.print_log_message('INFO', "orchestrator: mapping_create_indexes_and_constraints: Starting global recreate for indexes and constraints")
        target_conn = self.load_connector('target')
        target_conn.connect()

        target_constraints = self.migrator_tables.fetch_all_mapping_target_constraints()
        target_indexes = self.migrator_tables.fetch_all_mapping_target_indexes()
        constraint_index_info = {c['constraint_name']: c['constraint_type'] for c in target_constraints if c['constraint_type'] in ('UNIQUE', 'EXCLUSION')}

        # 1. Recreate indexes
        self.config_parser.print_log_message('INFO', "orchestrator: mapping_create_indexes_and_constraints: Recreating non-primary key indexes")
        for index in target_indexes:
            if not index.get('is_primary_key', False):
                if index['index_name'] not in constraint_index_info:
                    self.config_parser.print_log_message('DEBUG', f"orchestrator: Recreating index {index['index_name']} on {index['target_schema_name']}.{index['target_table_name']}")
                    try:
                        target_conn.execute_query(index['index_def'])
                        self.migrator_tables.update_mapping_target_index_status({'row_id': index['id'], 'success': True, 'message': 'migrated OK'})
                    except Exception as ex:
                        self.config_parser.print_log_message('ERROR', f"orchestrator: Error recreating index {index['index_name']}: {ex}")
                        self.migrator_tables.update_mapping_target_index_status({'row_id': index['id'], 'success': False, 'message': f'ERROR: {ex}'})
                else:
                    c_type = constraint_index_info[index['index_name']]
                    self.config_parser.print_log_message('DEBUG', f"orchestrator: Skipping index recreation {index['index_name']} on {index['target_schema_name']}.{index['target_table_name']} (handled by {c_type} constraint '{index['index_name']}')")

        # 2. Recreate constraints
        self.config_parser.print_log_message('INFO', "orchestrator: mapping_create_indexes_and_constraints: Recreating non-primary key constraints")
        for constraint in target_constraints:
            if constraint['constraint_type'] != 'PRIMARY KEY':
                self.config_parser.print_log_message('DEBUG', f"orchestrator: Recreating constraint {constraint['constraint_name']} on {constraint['target_schema_name']}.{constraint['target_table_name']}")
                add_sql = f'ALTER TABLE "{constraint["target_schema_name"]}"."{constraint["target_table_name"]}" ADD CONSTRAINT "{constraint["constraint_name"]}" {constraint["constraint_def"]}'
                try:
                    target_conn.execute_query(add_sql)
                    self.migrator_tables.update_mapping_target_constraint_status({'row_id': constraint['id'], 'success': True, 'message': 'migrated OK'})
                except Exception as ex:
                    self.config_parser.print_log_message('ERROR', f"orchestrator: Error recreating constraint {constraint['constraint_name']}: {ex}")
                    self.migrator_tables.update_mapping_target_constraint_status({'row_id': constraint['id'], 'success': False, 'message': f'ERROR: {ex}'})

        target_conn.disconnect()

    def mapping_check_indexes_and_constraints(self):
        self.config_parser.print_log_message('INFO', "orchestrator: mapping_check_indexes_and_constraints: Starting global verification of indexes and constraints")
        target_conn = self.load_connector('target')
        target_conn.connect()

        total_checked_constraints = 0
        total_missing_constraints = 0
        total_checked_indexes = 0
        total_missing_indexes = 0

        target_constraints = self.migrator_tables.fetch_all_mapping_target_constraints()
        target_indexes = self.migrator_tables.fetch_all_mapping_target_indexes()

        tables_to_check = set()
        for c in target_constraints:
            tables_to_check.add((c['target_schema_name'], c['target_table_name']))
        for i in target_indexes:
            tables_to_check.add((i['target_schema_name'], i['target_table_name']))

        for schema_name, table_name in tables_to_check:
            expected_constraints = [c for c in target_constraints if c['target_schema_name'] == schema_name and c['target_table_name'] == table_name]
            actual_constraints = target_conn.fetch_mapping_target_constraints(schema_name, table_name)
            actual_constraint_names = {c['constraint_name'] for c in actual_constraints}

            for expected in expected_constraints:
                total_checked_constraints += 1
                if expected['constraint_name'] not in actual_constraint_names:
                    total_missing_constraints += 1
                    self.config_parser.print_log_message('WARNING', f"orchestrator: Target database is missing constraint '{expected['constraint_name']}' on {schema_name}.{table_name}")

            expected_indexes = [i for i in target_indexes if i['target_schema_name'] == schema_name and i['target_table_name'] == table_name]
            actual_indexes = target_conn.fetch_mapping_target_indexes(schema_name, table_name)
            actual_index_names = {i['index_name'] for i in actual_indexes}

            for expected in expected_indexes:
                total_checked_indexes += 1
                if expected['index_name'] not in actual_index_names:
                    total_missing_indexes += 1
                    self.config_parser.print_log_message('WARNING', f"orchestrator: Target database is missing index '{expected['index_name']}' on {schema_name}.{table_name}")

        self.config_parser.print_log_message('INFO', f"orchestrator: mapping_check_indexes_and_constraints: Constraints Summary: {total_checked_constraints} checked, {total_missing_constraints} missing.")
        self.config_parser.print_log_message('INFO', f"orchestrator: mapping_check_indexes_and_constraints: Indexes Summary: {total_checked_indexes} checked, {total_missing_indexes} missing.")
        
        target_conn.disconnect()

    def mapping_copy_data(self):
        self.migrator_tables.insert_main({'task_name': 'Orchestrator', 'subtask_name': 'mapping data copy'})
        workers_requested = self.config_parser.get_parallel_workers_count()

        self.config_parser.print_log_message('INFO', f"orchestrator: mapping_copy_data: Starting {workers_requested} parallel workers for mapping data copy.")

        raw_tables = self.migrator_tables.fetch_all_tables(only_unfinished=False)
        migrate_tables = []
        for table_row in raw_tables:
            table_data = self.migrator_tables.decode_table_row(table_row)
            source_rows_all = table_data.get('source_table_rows_all', 0)
            source_rows_limited = table_data.get('source_table_rows_limited', 0)
            target_rows = table_data.get('target_table_rows', -1)

            conflict_action = 'skip'
            if source_rows_limited > 0 and target_rows > 0:
                conflict_action = self.config_parser.get_mapping_data_resolution(table_data['target_table_name'])
                table_data['data_conflict_action'] = conflict_action
                self.config_parser.print_log_message('INFO', f"orchestrator: mapping_copy_data: Table {table_data['target_table_name']} has conflict action: {conflict_action}")

            if source_rows_limited > 0 and (target_rows == 0 or conflict_action in ('replace', 'merge_keep_target', 'merge_keep_source')):
                target_columns = table_data.get('target_columns', {})
                insert_values = []
                keys_sorted = sorted(target_columns.keys(), key=lambda x: int(x))
                for col_key in keys_sorted:
                    col_data_type = target_columns[col_key].get('data_type', '')
                    if col_data_type:
                        if col_data_type.lower() in ('boolean', 'bool'):
                            insert_values.append(f'cast(%s as text)::{col_data_type}')
                        elif col_data_type.lower() == 'oid':
                            insert_values.append('lo_from_bytea(0, %s::bytea)')
                        else:
                            insert_values.append(f'%s::{col_data_type}')
                    else:
                        insert_values.append('%s')
                table_data['insert_values'] = ', '.join(insert_values)
                migrate_tables.append(table_data)
            else:
                protocol_id = self.migrator_tables.insert_data_migration({
                    'worker_id': None,
                    'source_table_id': table_data['source_table_id'],
                    'source_schema_name': table_data['source_schema_name'],
                    'source_table_name': table_data['source_table_name'],
                    'target_schema_name': table_data['target_schema_name'],
                    'target_table_name': table_data['target_table_name'],
                    'source_table_rows_all': source_rows_all,
                    'source_table_rows_limited': source_rows_limited,
                    'target_table_rows': target_rows if target_rows != -1 else 0,
                })

                if source_rows_limited == 0:
                    msg = 'Skipped (0 rows)'
                    status_msg = 'mapped data OK (0 rows)'
                else:
                    msg = f"Skipped (target_rows={target_rows}, action={conflict_action})"
                    status_msg = f"mapped data skipped (source_rows_limited={source_rows_limited}, target_rows={target_rows}, action={conflict_action})"

                self.migrator_tables.update_data_migration_status({
                    'row_id': protocol_id,
                    'success': True,
                    'message': msg,
                    'target_table_rows': target_rows if target_rows != -1 else 0,
                    'batch_count': 0,
                    'shortest_batch_seconds': 0,
                    'longest_batch_seconds': 0,
                    'average_batch_seconds': 0,
                })
                self.migrator_tables.update_table_status({'row_id': table_data['id'], 'success': True, 'message': status_msg})

        if len(migrate_tables) > 0:
            with concurrent.futures.ThreadPoolExecutor(max_workers=workers_requested) as executor:
                futures = {}
                for table_data in migrate_tables:
                    self.config_parser.print_log_message('DEBUG3', f"orchestrator: mapping_copy_data: futures running count: {len(futures)}")
                    while len(futures) >= workers_requested:
                        done, _ = concurrent.futures.wait(futures, return_when=concurrent.futures.FIRST_COMPLETED)
                        for future in done:
                            table_done = futures.pop(future)
                            if future.result() == False:
                                if self.on_error_action == 'stop':
                                    self.config_parser.print_log_message('ERROR', "orchestrator: mapping_copy_data: Stopping execution due to error.")
                                    exit(1)
                            else:
                                self.migrator_tables.update_table_status({'row_id': table_done['id'], 'success': True, 'message': 'mapped data OK'})

                    future = executor.submit(self.mapping_data_worker, table_data)
                    futures[future] = table_data

                self.config_parser.print_log_message('INFO', "orchestrator: mapping_copy_data: Processing remaining futures")
                for future in concurrent.futures.as_completed(futures):
                    table_done = futures[future]
                    if future.result() == False:
                        if self.on_error_action == 'stop':
                            self.config_parser.print_log_message('ERROR', "orchestrator: mapping_copy_data: Stopping execution due to error.")
                            exit(1)
                    else:
                        self.migrator_tables.update_table_status({'row_id': table_done['id'], 'success': True, 'message': 'mapped data OK'})

            self.config_parser.print_log_message('INFO', "orchestrator: mapping_copy_data: Tables data copied successfully.")
        else:
            self.config_parser.print_log_message('INFO', "orchestrator: mapping_copy_data: No tables met the criteria for data copy.")

        self.migrator_tables.update_main_status({'task_name': 'Orchestrator', 'subtask_name': 'mapping data copy', 'success': True, 'message': 'finished OK'})

    def mapping_data_worker(self, table_data):
        worker_id = uuid.uuid4()
        part_name = 'start'
        worker_source_connection = None
        worker_target_connection = None
        try:
            self.config_parser.print_log_message('INFO', f"orchestrator: mapping_data_worker: Worker {worker_id}: Processing table {table_data['target_table_name']}")

            worker_source_connection = self.load_connector('source')
            worker_target_connection = self.load_connector('target')

            part_name = 'connect source and target'
            worker_source_connection.connect()
            worker_target_connection.connect()

            if getattr(worker_source_connection, 'session_settings', None):
                worker_source_connection.execute_query(worker_source_connection.session_settings)

            if getattr(worker_target_connection, 'session_settings', None):
                worker_target_connection.execute_query(worker_target_connection.session_settings)

            chunk_size = -1
            try:
                chunk_size = self.config_parser.get_chunk_size()
            except Exception:
                pass

            settings = {
                'worker_id': worker_id,
                'source_schema_name': table_data['source_schema_name'],
                'source_table_name': table_data['source_table_name'],
                'source_table_id': table_data['source_table_id'],
                'source_columns': table_data['source_columns'],
                'target_schema_name': table_data['target_schema_name'],
                'target_table_name': table_data['target_table_name'],
                'target_columns': table_data['target_columns'],
                'insert_values': table_data.get('insert_values'),
                'primary_key_columns': self.migrator_tables.select_primary_key({'source_schema_name': table_data['source_schema_name'], 'source_table_name': table_data['source_table_name']}),
                'batch_size': self.config_parser.get_batch_size(),
                'migrator_tables': self.migrator_tables,
                'migration_limitation': None,
                'chunk_size': chunk_size,
                'chunk_number': 1,
                'resume_after_crash': False,
                'drop_unfinished_tables': False,
                'data_conflict_action': table_data.get('data_conflict_action')
            }

            if settings.get('data_conflict_action') == 'replace':
                self.config_parser.print_log_message('INFO', f"orchestrator: mapping_data_worker: Worker {worker_id}: Truncating table {table_data['target_table_name']} due to 'replace' conflict action.")
                worker_target_connection.execute_query(f'TRUNCATE TABLE "{table_data["target_schema_name"]}"."{table_data["target_table_name"]}"')
                settings['target_table_rows'] = 0

            part_name = 'migrate_table'
            rows_migrated = 0
            while True:
                if self.config_parser.pause_migration_fired():
                    self.config_parser.print_log_message('INFO', f"orchestrator: mapping_data_worker: Worker {worker_id}: Migration paused for table {table_data['target_table_name']}.")
                    self.config_parser.wait_for_resume()
                    self.config_parser.print_log_message('INFO', f"orchestrator: mapping_data_worker: Worker {worker_id}: Resuming migration for table {table_data['target_table_name']}.")

                migration_stats = worker_source_connection.migrate_table(worker_target_connection, settings)

                rows_migrated += migration_stats.get('rows_migrated', 0)
                if migration_stats.get('finished', True):
                    break
                settings['chunk_number'] += 1

            worker_source_connection.disconnect()

            # sequences setting
            target_schema_name = table_data['target_schema_name']
            target_table_name = table_data['target_table_name']
            part_name = 'sequences'
            self.config_parser.print_log_message('INFO', f"orchestrator: mapping_data_worker: Worker {worker_id}: Setting sequences for table {target_table_name} in target database.")
            sequences = worker_target_connection.fetch_table_sequences(target_schema_name, target_table_name)
            if sequences:
                for order_num, sequence_details in sequences.items():
                    sequence_id = sequence_details['id']
                    sequence_name = sequence_details['name']
                    column_name = sequence_details['column_name']
                    sequence_sql = sequence_details['set_sequence_sql']
                    self.migrator_tables.insert_sequence({
                        'sequence_id': sequence_id,
                        'target_schema_name': target_schema_name,
                        'target_table_name': target_table_name,
                        'target_column_name': column_name,
                        'target_sequence_name': sequence_name,
                        'target_sequence_sql': sequence_sql
                    })
                    self.config_parser.print_log_message( 'DEBUG', f"orchestrator: mapping_data_worker: Worker {worker_id}: Setting sequence with SQL: {sequence_sql}")
                    try:
                        worker_target_connection.execute_query(sequence_sql)
                        self.config_parser.print_log_message('INFO', f"orchestrator: mapping_data_worker: Worker {worker_id}: Sequence ({order_num}) {sequence_name} set successfully for table {target_table_name}.")
                        seq_curr_val = worker_target_connection.get_sequence_current_value(sequence_id)
                        self.config_parser.print_log_message('INFO', f"orchestrator: mapping_data_worker: Worker {worker_id}: Current value of sequence {sequence_name} is {seq_curr_val}.")
                        self.migrator_tables.update_sequence_status({'sequence_id': sequence_id, 'success': True, 'message': 'migrated OK'})
                    except Exception as e:
                        self.migrator_tables.update_sequence_status({'sequence_id': sequence_id, 'success': False, 'message': f'ERROR: {e}'})
                        self.migrator_tables.update_table_status({'row_id': table_data['id'], 'success': False, 'message': f'ERROR: {e}'})
                        self.config_parser.print_log_message('ERROR', f"orchestrator: mapping_data_worker: Worker {worker_id}: Error setting sequence {sequence_name} for table {target_table_name}: {e}")
            else:
                self.config_parser.print_log_message('INFO', f"orchestrator: mapping_data_worker: Worker {worker_id}: No sequences found for table {target_table_name}.")

            worker_target_connection.disconnect()

            return True

        except Exception as e:
            self.config_parser.print_log_message('ERROR', f"orchestrator: mapping_data_worker: Worker {worker_id}: Error during {part_name} -> {e}")
            return False

    def load_connector(self, source_or_target):
        """Dynamically load the database connector."""
        # Get the database type from the config
        database_type = self.config_parser.get_db_type(source_or_target)
        self.config_parser.print_log_message( 'DEBUG', f"orchestrator: load_connector: Loading connector for {source_or_target} with database type: {database_type}")
        if source_or_target == 'target' and database_type != 'postgresql':
            raise ValueError("Target database type must be 'postgresql'")
        # Check if the database type is supported
        database_module = MigratorConstants.get_modules().get(database_type)
        if not database_module:
            raise ValueError(f"Unsupported database type: {database_type}")
        # Import the module and get the class
        module_name, class_name = database_module.split(':')
        if not module_name or not class_name:
            raise ValueError(f"Invalid module format: {database_module}")
        # Import the module and get the class
        module = importlib.import_module(module_name)
        connector_class = getattr(module, class_name)
        return connector_class(self.config_parser, source_or_target)

    def run_post_migration_script(self):
        post_migration_script = self.config_parser.get_post_migration_script()
        if post_migration_script:
            self.config_parser.print_log_message('INFO', "orchestrator: run_post_migration_script: Running post-migration script in target database.")
            try:
                self.target_connection.connect()
                self.target_connection.execute_sql_script(post_migration_script)
                self.target_connection.disconnect()
                self.config_parser.print_log_message('INFO', "orchestrator: run_post_migration_script: Post-migration script executed successfully.")
            except Exception as e:
                self.handle_error(e, 'post-migration script')

    def stdwf_migrate_tables(self):
        self.migrator_tables.insert_main({'task_name': 'Orchestrator', 'subtask_name': 'tables migration'})
        workers_requested = self.config_parser.get_parallel_workers_count()
        settings = {
            'source_db_type': self.config_parser.get_source_db_type(),
            'target_db_type': self.config_parser.get_target_db_type(),
            'create_tables': self.config_parser.should_create_tables(),
            'drop_tables': self.config_parser.should_drop_tables(),
            'truncate_tables': self.config_parser.should_truncate_tables(),
            'migrate_data': self.config_parser.should_migrate_data(),
            'batch_size': self.config_parser.get_batch_size(),
            'migrator_tables': self.migrator_tables,
            'resume_after_crash': self.config_parser.is_resume_after_crash(),
            'drop_unfinished_tables': self.config_parser.should_drop_unfinished_tables(),
        }

        self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_tables: Starting {workers_requested} parallel workers to create tables in target database.")

        ## in case of crash recovery, we only migrate tables that are not yet migrated -> success is not True
        migrate_tables = self.migrator_tables.fetch_all_tables(self.config_parser.is_resume_after_crash())

        if len(migrate_tables) > 0:
            target_connection = self.load_connector('target')
            target_connection.connect()

            failed_tables = set()
            for table_row in migrate_tables:
                table_data = self.migrator_tables.decode_table_row(table_row)
                target_schema_name = table_data['target_schema_name']
                target_table_name = self.config_parser.convert_names_case(table_data['target_table_name'])

                if ((settings['drop_tables'] and not settings['resume_after_crash'])
                    or (settings['resume_after_crash'] and settings['drop_unfinished_tables'])
                    or (settings['resume_after_crash'] and not target_connection.target_table_exists(target_schema_name, target_table_name))):

                    repeat_count = 0
                    drop_query = f'DROP TABLE IF EXISTS "{target_schema_name}"."{target_table_name}" CASCADE'
                    self.config_parser.print_log_message('DEBUG', f"orchestrator: run_migrate_tables: Dropping table {target_table_name} - Query: {drop_query}")
                    drop_success = False
                    while True:
                        try:
                            target_connection.execute_query(drop_query)
                            drop_success = True
                            break
                        except Exception as e:
                            if repeat_count > 5:
                                self.config_parser.print_log_message('ERROR', f"orchestrator: run_migrate_tables: Error dropping table {target_table_name}: {e}")
                                self.migrator_tables.update_table_status({'row_id': table_data['id'], 'success': False, 'message': f'ERROR: {e}'})
                                failed_tables.add(table_data['id'])
                                break
                            else:
                                repeat_count += 1
                                self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_tables: Retrying to drop table {target_table_name} ({repeat_count})...")
                                time.sleep(10)
                    if drop_success:
                        self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_tables: Table '{target_table_name}' dropped successfully.")

            try:
                target_connection.disconnect()
            except Exception:
                pass

            migrate_tables = [t for t in migrate_tables if self.migrator_tables.decode_table_row(t)['id'] not in failed_tables]

            if len(migrate_tables) == 0 and failed_tables and self.on_error_action == 'stop':
                self.config_parser.print_log_message('ERROR', "orchestrator: run_migrate_tables: Stopping execution due to error during table drops.")
                exit(1)

        if len(migrate_tables) > 0:
            with concurrent.futures.ThreadPoolExecutor(max_workers=workers_requested) as executor:
                futures = {}
                for table_row in migrate_tables:
                    table_data = self.migrator_tables.decode_table_row(table_row)
                    # (table_data['primary_key_columns'],
                    # table_data['primary_key_columns_count'],
                    # table_data['primary_key_columns_types']) = self.migrator_tables.select_primary_key(table_data['target_schema_name'], table_data['target_table_name'])
                    # Submit tasks until we have workers_requested running, then as soon as one finishes, submit a new one
                    self.config_parser.print_log_message('DEBUG3', f"orchestrator: run_migrate_tables: futures running count: {len(futures)}")
                    while len(futures) >= workers_requested:
                        # Wait for the first completed future
                        done, _ = concurrent.futures.wait(futures, return_when=concurrent.futures.FIRST_COMPLETED)
                        for future in done:
                            table_done = futures.pop(future)
                            if future.result() == False:
                                if self.on_error_action == 'stop':
                                    self.config_parser.print_log_message('ERROR', "orchestrator: run_migrate_tables: Stopping execution due to error.")
                                    exit(1)
                            else:
                                self.migrator_tables.update_table_status({'row_id': table_done['id'], 'success': True, 'message': 'migrated OK'})

                    # Submit the next task
                    future = executor.submit(self.table_worker, table_data, settings)
                    futures[future] = table_data

                # Process remaining futures
                self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_tables: Processing remaining futures")
                for future in concurrent.futures.as_completed(futures):
                    table_done = futures[future]
                    if future.result() == False:
                        if self.on_error_action == 'stop':
                            self.config_parser.print_log_message('ERROR', "orchestrator: run_migrate_tables: Stopping execution due to error.")
                            exit(1)
                    else:
                        self.migrator_tables.update_table_status({'row_id': table_done['id'], 'success': True, 'message': 'migrated OK'})

            self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_tables: Tables processed successfully.")
        else:
            self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_tables: No tables to create.")

        self.migrator_tables.update_main_status({'task_name': 'Orchestrator', 'subtask_name': 'tables migration', 'success': True, 'message': 'finished OK'})

    def run_create_user_defined_types(self):
        self.migrator_tables.insert_main({'task_name': 'Orchestrator', 'subtask_name': 'user defined types migration'})
        self.config_parser.print_log_message('INFO', "orchestrator: run_create_user_defined_types: Migrating user defined types.")
        user_defined_types = self.migrator_tables.fetch_all_user_defined_types()
        if len(user_defined_types) > 0:
            for type_row in user_defined_types:
                type_data = self.migrator_tables.decode_user_defined_type_row(type_row)
                self.migrator_tables.update_protocol_task_started('user_defined_types', type_data['id'])
                self.config_parser.print_log_message('INFO', f"orchestrator: run_create_user_defined_types: Creating user defined type {type_data['target_type_name']} in target database.")
                self.config_parser.print_log_message('DEBUG3', f"orchestrator: run_create_user_defined_types: type_data: {type_data['target_type_sql']}")
                try:
                    self.target_connection.connect()

                    # Check if domain already exists
                    check_query = """SELECT data_type FROM information_schema.domains WHERE domain_schema = %s AND domain_name = %s"""
                    cursor = self.target_connection.connection.cursor()
                    cursor.execute(check_query, (type_data['target_schema_name'], type_data['target_type_name']))
                    existing = cursor.fetchone()
                    cursor.close()

                    if existing:
                        existing_type = existing[0]
                        target_basic_type = type_data.get('target_basic_type')

                        # Normalize and compare
                        norm_existing = self._normalize_data_type(existing_type)
                        norm_target = self._normalize_data_type(target_basic_type)

                        if target_basic_type and norm_existing == norm_target:
                            msg = f"Domain {type_data['target_type_name']} with underlying type {target_basic_type} already exists (normalized match: {norm_existing}). Skipping."
                            self.config_parser.print_log_message('INFO', msg)
                            self.migrator_tables.update_user_defined_type_status({'row_id': type_data['id'], 'success': True, 'message': 'skipped (exists)'})
                        else:
                            msg = f"Domain {type_data['target_type_name']} already exists but with different underlying type: {existing_type} (expected: {target_basic_type}). Normalized: {norm_existing} vs {norm_target}. Skipping creation."
                            self.config_parser.print_log_message('ERROR', msg)
                            self.migrator_tables.update_user_defined_type_status({'row_id': type_data['id'], 'success': False, 'message': f'ERROR: {msg}'})
                    else:
                        self.target_connection.execute_query(type_data['target_type_sql'])
                        self.migrator_tables.update_user_defined_type_status({'row_id': type_data['id'], 'success': True, 'message': 'migrated OK'})
                        self.config_parser.print_log_message('INFO', f"orchestrator: run_create_user_defined_types: User defined type {type_data['target_type_name']} created successfully.")

                    self.target_connection.disconnect()
                except Exception as e:
                    self.migrator_tables.update_user_defined_type_status({'row_id': type_data['id'], 'success': False, 'message': f'ERROR: {e}'})
                    self.handle_error(e, f"create_user_defined_type {type_data['target_type_name']}")
            self.config_parser.print_log_message('INFO', "orchestrator: run_create_user_defined_types: User defined types migrated successfully.")
        else:
            self.config_parser.print_log_message('INFO', "orchestrator: run_create_user_defined_types: No user defined types found to migrate.")
        self.migrator_tables.update_main_status({'task_name': 'Orchestrator', 'subtask_name': 'user defined types migration', 'success': True, 'message': 'finished OK'})

    def _normalize_data_type(self, data_type):
        """Normalize PostgreSQL data type names for comparison."""
        if not data_type:
            return None

        dt = data_type.lower().strip()

        # Mapping of aliases to canonical/common names
        mapping = {
            'character varying': 'varchar',
            'character': 'char',
            'integer': 'int',
            'int4': 'int',
            'bigint': 'int8',
            'smallint': 'int2',
            'decimal': 'numeric',
            'real': 'float4',
            'double precision': 'float8',
            'boolean': 'bool',
            'timestamp without time zone': 'timestamp',
            'timestamp with time zone': 'timestamptz',
            'time without time zone': 'time',
            'time with time zone': 'timetz'
        }

        return mapping.get(dt, dt)

    def run_create_domains(self):
        self.migrator_tables.insert_main({'task_name': 'Orchestrator', 'subtask_name': 'domains migration'})
        self.config_parser.print_log_message('INFO', "orchestrator: run_create_domains: Migrating domains.")
        domains = self.migrator_tables.fetch_all_domains({})
        if len(domains) > 0:
            for domain_row in domains:
                domain_data = self.migrator_tables.decode_domain_row(domain_row)
                self.migrator_tables.update_protocol_task_started('domains', domain_data['id'])
                self.config_parser.print_log_message('INFO', f"orchestrator: run_create_domains: Creating domain {domain_data['target_domain_name']} in target database using SQL: {domain_data['target_domain_sql']}")
                try:
                    self.target_connection.connect()
                    self.target_connection.execute_query(domain_data['target_domain_sql'])
                    self.migrator_tables.update_domain_status({'row_id': domain_data['id'], 'success': True, 'message': 'migrated OK'})
                    self.config_parser.print_log_message('INFO', f"orchestrator: run_create_domains: Domain {domain_data['target_domain_name']} created successfully.")
                    self.target_connection.disconnect()
                except Exception as e:
                    self.migrator_tables.update_domain_status({'row_id': domain_data['id'], 'success': False, 'message': f'ERROR: {e}'})
                    self.handle_error(e, f"create_domain {domain_data['target_domain_name']}")
            self.config_parser.print_log_message('INFO', "orchestrator: run_create_domains: Domains migrated successfully.")
        else:
            self.config_parser.print_log_message('INFO', "orchestrator: run_create_domains: No domains found to migrate.")
        self.migrator_tables.update_main_status({'task_name': 'Orchestrator', 'subtask_name': 'domains migration', 'success': True, 'message': 'finished OK'})

    def stdwf_migrate_indexes(self, run_mode='standard'):
        self.migrator_tables.insert_main({'task_name': 'Orchestrator', 'subtask_name': 'indexes migration'})
        workers_requested = self.config_parser.get_parallel_workers_count()
        target_db_type = self.config_parser.get_target_db_type()

        self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_indexes: Starting {workers_requested} parallel workers to create indexes in target database.")
        migrate_indexes = self.migrator_tables.fetch_all_indexes()
        if len(migrate_indexes) > 0:
            with concurrent.futures.ThreadPoolExecutor(max_workers=workers_requested) as executor:
                futures = {}
                for index_row in migrate_indexes:
                    index_data = self.migrator_tables.decode_index_row(index_row)

                    if run_mode == 'function_based' and not index_data['is_function_based']:
                        self.config_parser.print_log_message( 'DEBUG3', f"orchestrator: run_migrate_indexes: Function based run mode: Skipping index {index_data['index_name']} as it is not a function based index.")
                        continue
                    elif run_mode == 'standard' and index_data['is_function_based']:
                        self.config_parser.print_log_message( 'INFO', f"orchestrator: run_migrate_indexes: Standard run mode: Skipping function based index {index_data['index_name']} ")
                        continue
                    if not self.config_parser.should_migrate_indexes(index_data['source_table_name']):
                        self.config_parser.print_log_message('DEBUG3', f"orchestrator: run_migrate_indexes: Skipping index {index_data['index_name']} as it is not configured for migration.")
                        continue

                    self.config_parser.print_log_message('DEBUG3', f"orchestrator: run_migrate_indexes: futures running count: {len(futures)}")
                    while len(futures) >= workers_requested:
                        done, _ = concurrent.futures.wait(futures, return_when=concurrent.futures.FIRST_COMPLETED)
                        for future in done:
                            index_done = futures.pop(future)
                            if future.result() == False:
                                if self.on_error_action == 'stop':
                                    self.config_parser.print_log_message('ERROR', "orchestrator: run_migrate_indexes: Stopping execution due to error.")
                                    exit(1)
                            else:
                                self.migrator_tables.update_index_status({'row_id': index_done['id'], 'success': True, 'message': 'migrated OK'})

                    # Submit the next task
                    future = executor.submit(self.index_worker, index_data, target_db_type)
                    futures[future] = index_data

                # Process remaining futures
                for future in concurrent.futures.as_completed(futures):
                    index_done = futures[future]
                    if future.result() == False:
                        if self.on_error_action == 'stop':
                            self.config_parser.print_log_message('ERROR', "orchestrator: run_migrate_indexes: Stopping execution due to error.")
                            exit(1)
                    else:
                        self.migrator_tables.update_index_status({'row_id': index_done['id'], 'success': True, 'message': 'migrated OK'})

            self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_indexes: Indexes processed successfully.")
        else:
            self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_indexes: No indexes to create.")

        self.migrator_tables.update_main_status({'task_name': 'Orchestrator', 'subtask_name': 'indexes migration', 'success': True, 'message': 'finished OK'})

    def stdwf_migrate_constraints(self):
        self.migrator_tables.insert_main({'task_name': 'Orchestrator', 'subtask_name': 'constraints migration'})
        workers_requested = self.config_parser.get_parallel_workers_count()
        target_db_type = self.config_parser.get_target_db_type()

        self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_constraints: Starting {workers_requested} parallel workers to create constraints in target database.")
        migrate_constraints = self.migrator_tables.fetch_all_constraints()
        if len(migrate_constraints) > 0:
            with concurrent.futures.ThreadPoolExecutor(max_workers=workers_requested) as executor:
                futures = {}
                for constraint_row in migrate_constraints:
                    constraint_data = self.migrator_tables.decode_constraint_row(constraint_row)
                    self.config_parser.print_log_message('DEBUG3', f"orchestrator: run_migrate_constraints: futures running count: {len(futures)}")

                    if not self.config_parser.should_migrate_constraints(constraint_data['source_table_name']):
                        self.config_parser.print_log_message('DEBUG3', f"orchestrator: run_migrate_constraints: Skipping constraint {constraint_data['constraint_name']} as it is not configured for migration.")
                        continue

                    while len(futures) >= workers_requested:
                        done, _ = concurrent.futures.wait(futures, return_when=concurrent.futures.FIRST_COMPLETED)
                        for future in done:
                            constraint_done = futures.pop(future)
                            if future.result() == False:
                                if self.on_error_action == 'stop':
                                    self.config_parser.print_log_message('ERROR', "orchestrator: run_migrate_constraints: Stopping execution due to error.")
                                    exit(1)
                            else:
                                self.migrator_tables.update_constraint_status({'row_id': constraint_done['id'], 'success': True, 'message': 'migrated OK'})
                    # Submit the next task
                    future = executor.submit(self.constraint_worker, constraint_data, target_db_type)
                    futures[future] = constraint_data

                # Process remaining futures
                for future in concurrent.futures.as_completed(futures):
                    constraint_done = futures[future]
                    if future.result() == False:
                        if self.on_error_action == 'stop':
                            self.config_parser.print_log_message('ERROR', "orchestrator: run_migrate_constraints: Stopping execution due to error.")
                            exit(1)
                    else:
                        self.migrator_tables.update_constraint_status({'row_id': constraint_done['id'], 'success': True, 'message': 'migrated OK'})

            self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_constraints: Constraints processed successfully.")
        else:
            self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_constraints: No constraints to create.")

        self.migrator_tables.update_main_status({'task_name': 'Orchestrator', 'subtask_name': 'constraints migration', 'success': True, 'message': 'finished OK'})

    def table_worker(self, table_data, settings):
        worker_id = uuid.uuid4()
        part_name = 'start'
        worker_source_connection = None
        worker_target_connection = None
        rows_migrated = 0
        worker_start_time = time.time()
        worker_end_time = None
        try:
            # target_schema_name = self.config_parser.convert_names_case(table_data['target_schema_name'])
            target_schema_name = table_data['target_schema_name'] ## target schema is used as defined in the config file, no case conversion
            target_table_name = self.config_parser.convert_names_case(table_data['target_table_name'])
            create_table_sql = table_data['target_table_sql']
            migrator_tables = settings['migrator_tables']

            source_table_rows_all = table_data.get('source_table_rows_all', 0)
            source_table_rows_limited = table_data.get('source_table_rows_limited', 0)
            target_table_rows = 0

            if create_table_sql is None:
                self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Table {target_table_name} does not have a CREATE TABLE statement - skipping.")
                return False

            self.migrator_tables.update_protocol_task_started('tables', table_data['id'])
            self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Creating table {target_table_name} in target database ({settings['source_db_type']}:{settings['target_db_type']}-{settings['drop_tables']}/{settings['truncate_tables']}/{settings['create_tables']}/{settings['migrate_data']}).")

            # Each worker uses its own separate connection to the target database
            if settings['target_db_type'] == 'postgresql':
                worker_target_connection = self.load_connector('target')
            else:
                raise ValueError(f"Unsupported target database type: {settings['target_db_type']}")

            part_name = 'connect target'
            worker_target_connection.connect()

            if worker_target_connection.session_settings:
                self.config_parser.print_log_message( 'DEBUG', f"orchestrator: table_worker: Worker {worker_id}: Executing session settings: {worker_target_connection.session_settings}")
                worker_target_connection.execute_query(worker_target_connection.session_settings)

            if ((settings['create_tables'] and not settings['resume_after_crash'])
                or (settings['resume_after_crash'] and settings['drop_unfinished_tables'])
                or (settings['resume_after_crash'] and not worker_target_connection.target_table_exists(target_schema_name, target_table_name))):
                self.config_parser.print_log_message( 'DEBUG', f"orchestrator: table_worker: Worker {worker_id}: Creating table with SQL: {create_table_sql}")
                part_name = 'create table'
                worker_target_connection.execute_query(create_table_sql)
                self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Table '{target_table_name}' created successfully.")

                if table_data.get('create_partitions_sql'):
                    part_name = 'create partitions'
                    self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Creating partitions for table {target_table_name} in target database.")

                    table_create_partitions_sql = json.loads(table_data['create_partitions_sql'])
                    for partition_sql in table_create_partitions_sql:
                        self.config_parser.print_log_message( 'DEBUG', f"orchestrator: table_worker: Worker {worker_id}: Creating partition for table {target_table_name}: {partition_sql}")
                        worker_target_connection.execute_query(partition_sql)
                        self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Partition of '{target_table_name}' created successfully [{partition_sql}].")

                ## now check alterations of columns due to FK IDENTITY dependency
                for result in migrator_tables.fk_find_dependent_columns_to_alter({
                    'target_schema_name': target_schema_name,
                    'target_table_name': target_table_name,
                }):
                    self.config_parser.print_log_message( 'DEBUG', f"orchestrator: table_worker: Worker {worker_id}: Found dependency for column alteration: {result}")
                    alter_column_sql = f"""
                        ALTER TABLE "{target_schema_name}"."{self.config_parser.convert_names_case(target_table_name)}"
                        ALTER COLUMN "{self.config_parser.convert_names_case(result['target_column'].replace('"',''))}"
                        TYPE {result['altered_data_type']}"""
                    self.config_parser.print_log_message( 'DEBUG', f"orchestrator: table_worker: Worker {worker_id}: Altering column with SQL: {alter_column_sql}")
                    worker_target_connection.execute_query(alter_column_sql)
                    self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Column '{result['target_column']}' altered successfully.")

            if ((settings['truncate_tables'] and not settings['resume_after_crash'])
                or (settings['resume_after_crash'] and settings['drop_unfinished_tables']) ):
                self.config_parser.print_log_message( 'DEBUG', f"orchestrator: table_worker: Worker {worker_id}: Truncating table {target_table_name}...")
                part_name = 'truncate table'
                repeat_count = 0
                ## Retry truncating the table if it fails due to locks or other issues
                while True:
                    try:
                        worker_target_connection.execute_query(f'''TRUNCATE TABLE "{target_schema_name}"."{target_table_name}" CASCADE''')
                        break
                    except Exception as e:
                        if repeat_count > 5:
                            self.config_parser.print_log_message('ERROR', f"orchestrator: table_worker: Worker {worker_id}: Error truncating table {target_table_name}: {e}")
                            self.migrator_tables.update_table_status({'row_id': table_data['id'], 'success': False, 'message': f'ERROR: {e}'})
                            return False
                        else:
                            repeat_count += 1
                            self.config_parser.print_log_message('ERROR', f"orchestrator: table_worker: Worker {worker_id}: Error in {repeat_count} attempt to truncate table {target_table_name}: {e}")
                            self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Retrying to truncate table {target_table_name} ({repeat_count})...")
                            part_name = f'retry truncate table ({repeat_count})'
                            time.sleep(10)
                self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Table '{target_table_name}' truncated successfully.")

            if self.config_parser.should_migrate_data(table_data['source_table_name']):
                # data migration

                part_name = 'connect source'
                worker_source_connection = self.load_connector('source')
                worker_source_connection.connect()

                data_source = self.migrator_tables.fetch_data_source({'source_schema_name': table_data['source_schema_name'], 'source_table_name': table_data['source_table_name']})
                self.config_parser.print_log_message('DEBUG3', f"orchestrator: table_worker: Worker {worker_id}: Checking data source for table {table_data['source_schema_name']}.{table_data['source_table_name']}: {data_source}")

                use_source_table = False

                if data_source is not None:
                    self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Data source for table {table_data['source_schema_name']}.{table_data['source_table_name']} is {data_source}.")
                    clean_objects = self.config_parser.get_source_data_export_clean()

                    if data_source['file_found'] and data_source['file_name'] is not None:
                        self.config_parser.print_log_message('DEBUG', f"orchestrator: table_worker: Worker {worker_id}: Data source file found: {data_source['file_name']} - proceeding with data migration.")

                        copy_command = f"""COPY "{target_schema_name}"."{target_table_name}"
                            FROM STDIN WITH
                            (FORMAT CSV, HEADER {data_source['format_options']['header']},
                            NULL '\\N',
                            DELIMITER '{data_source['format_options']['delimiter']}')"""
                            ## , QUOTE '{data_source['format_options']['quote']}'
                        self.config_parser.print_log_message('DEBUG3', f"orchestrator: table_worker: Worker {worker_id}: Copy command: {copy_command}")

                        ## for other formats relevant to other databases we simply add new data types
                        ## CSV format is common for all databases, but might be necessary to convert it to PostgreSQL CSV conventions
                        if data_source['format_options']['format'].upper() in ('CSV', 'UNL'):

                            self.config_parser.print_log_message('DEBUG3', f"orchestrator: table_worker: Worker {worker_id}: Data source format is CSV or UNL - proceeding with data migration.")
                            protocol_id = migrator_tables.insert_data_migration({
                                'worker_id': worker_id,
                                'source_table_id': table_data['source_table_id'],
                                'source_schema_name': table_data['source_schema_name'],
                                'source_table_name': table_data['source_table_name'],
                                'target_schema_name': table_data['target_schema_name'],
                                'target_table_name': table_data['target_table_name'],
                                'source_table_rows_all': source_table_rows_all,
                                'source_table_rows_limited': source_table_rows_limited,
                                'target_table_rows': target_table_rows,
                                })

                            if data_source['file_size'] == 0:
                                self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Data source for table {target_table_name} is {data_source['format_options']['format'].upper()} format, but file size is 0. Skipping data migration.")
                                # self.migrator_tables.update_table_status(table_data['id'], True, 'migrated OK (0 rows)')

                                migrator_tables.update_data_migration_status({
                                    'row_id': protocol_id,
                                    'success': True,
                                    'message': 'OK',
                                    'target_table_rows': target_table_rows,
                                    'batch_count': 0,
                                    'shortest_batch_seconds': 0,
                                    'longest_batch_seconds': 0,
                                    'average_batch_seconds': 0,
                                })
                            else:
                                migrator_tables.update_data_migration_started(protocol_id)
                                self.config_parser.print_log_message('DEBUG3', f"orchestrator: table_worker: Worker {worker_id}: Data source format is {data_source['format_options']['format'].upper()} - size: {data_source['file_size']} - proceeding with data migration.")
                                data_import_start_time = time.time()
                                source_files_to_process = []
                                converted_files_to_process = []

                                ## Split of big files is currently implemented only for Informix UNL data files
                                ## And even here it can be not efficient if client uses slow disk(s)
                                if self.config_parser.get_source_db_type() == 'informix' and data_source['format_options']['format'].upper() == 'UNL':
                                    big_files_split_enabled = self.config_parser.get_source_data_export_big_files_split_enabled()
                                    data_source_file_size = data_source['file_size']

                                    data_source_file_size_str = ""
                                    if data_source_file_size is not None:
                                        data_source_file_size_str = f"{data_source_file_size} B ({data_source_file_size / (1024 ** 3):.2f} GB)"
                                    split_threshold_bytes = self.config_parser.get_source_data_export_big_files_split_threshold_bytes()
                                    split_threshold_bytes_str = ""
                                    if split_threshold_bytes is not None:
                                        split_threshold_bytes_str = f"{split_threshold_bytes} B ({split_threshold_bytes / (1024 ** 3):.2f} GB)"

                                    self.config_parser.print_log_message('DEBUG', f"orchestrator: table_worker: Worker {worker_id}: Table: {target_table_name}: Data source file size: {data_source_file_size_str}, split threshold: {split_threshold_bytes_str}, big files split enabled: {big_files_split_enabled}")

                                    if big_files_split_enabled and data_source_file_size > split_threshold_bytes:
                                        # Big files split enabled and file size exceeds threshold
                                        self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: Data source for table {target_table_name} is a big file ({data_source_file_size} bytes). Splitting into smaller files.")
                                        source_files_to_process, converted_files_to_process = self.config_parser.split_big_unl_file(data_source)
                                        self.config_parser.print_log_message('DEBUG', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: Split source files: {source_files_to_process}, converted target file names: {converted_files_to_process}")

                                else:
                                    # Single file processing
                                    source_files_to_process.append(data_source['file_name'])
                                    converted_files_to_process.append(data_source['converted_file_name'])

                                data_source_settings = data_source.copy()
                                csv_file_name = None

                                self.config_parser.print_log_message('DEBUG3', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: Processing {len(source_files_to_process)} source files.")

                                for source_file_index, source_file_name in enumerate(source_files_to_process):
                                    data_source_settings['file_name'] = source_file_name
                                    data_source_settings['converted_file_name'] = converted_files_to_process[source_file_index]
                                    self.config_parser.print_log_message('DEBUG', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: Processing source file: {data_source_settings['file_name']}")

                                    if data_source_settings['format_options']['format'].upper() == 'UNL':
                                        # UNL data source - must be converted to CSV first
                                        self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: Data source for table {target_table_name} is UNL format. Converting to CSV.")
                                        part_name = 'convert UNL to CSV'
                                        csv_file_name = data_source_settings['converted_file_name']
                                        self.config_parser.convert_unl_to_csv(data_source_settings, table_data['source_columns'], table_data['target_columns'])
                                        csv_file_name = data_source_settings['converted_file_name']

                                    elif data_source_settings['format_options']['format'].upper() == 'CSV':
                                        self.config_parser.print_log_message('DEBUG3', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: Data source for table {target_table_name} is CSV format. Char set: {data_source_settings['format_options']['character_set']}")
                                        # Always convert CSV to UTF-8 to apply DB2 timestamp fixes and null replacements
                                        self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: Converting CSV to UTF-8 and applying DB2 fixes.")
                                        self.config_parser.convert_csv_to_utf8(data_source_settings, table_data.get('source_columns'), table_data.get('target_columns'))

                                        # CSV data source - use the converted file
                                        part_name = 'use CSV'
                                        csv_file_name = data_source_settings['converted_file_name']
                                        self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Data source for table {target_table_name} is CSV format. Using converted file {csv_file_name}.")

                                    try:
                                        if data_source_settings['lob_columns'] != '' and self.config_parser.should_migrate_lob_values():

                                            # Loop over import table row by row, select all columns, find column(s) specified in lob_columns and read their content
                                            lob_columns = [col.strip() for col in data_source_settings['lob_columns'].split(',') if col.strip()]
                                            lob_columns_count = len(lob_columns)

                                            # we need to have a primary key for LOB migration if multiple LOB columns are defined on the table - to check if row exists and to update only LOB column(s)
                                            # primary key unfortunately slows down the LOB migration due to index rebuilds, but is necessary for data integrity
                                            if lob_columns_count > 1:
                                                self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name} has multiple LOB columns defined ({data_source_settings['lob_columns']}) - adding primary key for LOB migration.")
                                                part_name = 'migrate LOBs - add primary key'
                                                primary_key_data = migrator_tables.select_primary_key_all_columns({'source_schema_name': table_data['source_schema_name'], 'source_table_name': table_data['source_table_name']})
                                                if primary_key_data is None:
                                                    self.config_parser.print_log_message('ERROR', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name} has LOB columns defined ({data_source_settings['lob_columns']}), but no primary key could be determined.")
                                                else:
                                                    self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name} has LOB columns defined ({data_source_settings['lob_columns']}). Adding primary key: {primary_key_data}")
                                                    try:
                                                        result = self.index_worker(primary_key_data, '')
                                                        if not result:
                                                            self.config_parser.print_log_message('ERROR', f"orchestrator: table_worker: Worker {worker_id}: Failed to add primary key to table {target_table_name} for LOB migration.")
                                                        else:
                                                            self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Primary key added successfully to table {target_table_name} for LOB migration.")
                                                    except Exception as e:
                                                        self.config_parser.print_log_message('ERROR', f"orchestrator: table_worker: Worker {worker_id}: Exception occurred while adding primary key to table {target_table_name} for LOB migration: {e}")
                                                        self.config_parser.print_log_message('ERROR', f"orchestrator: table_worker: Worker {worker_id}: Full error details: {traceback.format_exc()}")

                                            part_name = 'migrate LOBs start'

                                            table_name_for_lob_import = self.config_parser.get_table_name_for_lob_import(target_table_name)
                                            # LOB data migration - use the file directly
                                            self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Data source for table {target_table_name} has LOB columns: {data_source_settings['lob_columns']}. Migrating LOB data.")

                                            # drop table for lob import if exists
                                            drop_import_table_sql = f'DROP TABLE IF EXISTS "{target_schema_name}"."{table_name_for_lob_import}" CASCADE'
                                            worker_target_connection.execute_query(drop_import_table_sql)
                                            self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Intermediate import table {table_name_for_lob_import} dropped successfully.")

                                            # create intermediate table for rows without LOB data
                                            create_import_table_sql = re.sub(
                                                target_table_name,
                                                f"{table_name_for_lob_import}",
                                                table_data['target_table_sql'],
                                                flags=re.IGNORECASE
                                            )
                                            create_import_table_sql = re.sub( 'bytea', 'text', create_import_table_sql, flags=re.IGNORECASE)
                                            self.config_parser.print_log_message('DEBUG', f"orchestrator: table_worker: Worker {worker_id}: Creating intermediate import table with SQL: {create_import_table_sql}")
                                            worker_target_connection.execute_query(create_import_table_sql)

                                            # importing CSV (converted UNL) data into intermediate table
                                            imp_table_copy_command = f"""COPY "{target_schema_name}"."{table_name_for_lob_import}" FROM STDIN WITH
                                                (FORMAT CSV, HEADER {data_source_settings['format_options']['header']},
                                                NULL '\\N',
                                                DELIMITER '{data_source_settings['format_options']['delimiter']}')"""
                                            part_name = 'import to intermediate table'
                                            worker_target_connection.copy_from_file(imp_table_copy_command, csv_file_name)

                                            lob_col_name = None
                                            lob_col_index = None
                                            lob_col_type = None

                                            for lob_col in lob_columns:
                                                lob_col_name = lob_col
                                                for lob_col_ind, col_info in table_data['target_columns'].items():
                                                    if col_info['column_name'].lower() == lob_col.lower():
                                                        lob_col_index = int(lob_col_ind)
                                                        lob_col_type = col_info['data_type']
                                                        break

                                                part_name = f'table {target_table_name} process LOB column {lob_col_name}/{lob_col_index}'
                                                if lob_col_name is not None and lob_col_index is not None:
                                                    self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: LOB column {lob_col_name} found in target table {target_table_name} - index: {lob_col_index}, type: {lob_col_type}")

                                                    select_datafiles_sql = f"""SELECT DISTINCT coalesce(split_part({lob_col_name},',',3), '0,0,0') as datafile, count(*) as occurrences FROM "{target_schema_name}"."{table_name_for_lob_import}" group by 1 order by 1;"""

                                                    datafiles_cursor = worker_target_connection.connection.cursor()
                                                    datafiles_cursor.execute(select_datafiles_sql)
                                                    datafiles = datafiles_cursor.fetchall()
                                                    datafiles_cursor.close()
                                                    self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: Found {len(datafiles)} distinct data files for LOB column {lob_col_name}")

                                                    max_lob_parallel_workers = self.config_parser.get_source_data_export_workers()
                                                    if max_lob_parallel_workers <= 1:
                                                        max_lob_parallel_workers = 1
                                                    else:
                                                        if len(datafiles) <= 10:
                                                            max_lob_parallel_workers = 1
                                                            self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: Set max LOB parallel workers to {max_lob_parallel_workers} due to low data file count ({len(datafiles)})")
                                                        if len(datafiles) > 10 and len(datafiles) <= 50:
                                                            max_lob_parallel_workers = 2
                                                            self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: Set max LOB parallel workers to {max_lob_parallel_workers} due to medium data file count ({len(datafiles)})")
                                                        if len(datafiles) > 200:
                                                            max_lob_parallel_workers = max_lob_parallel_workers * 2
                                                            self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: Set max LOB parallel workers to {max_lob_parallel_workers} due to high data file count ({len(datafiles)})")

                                                    if len(datafiles) > 0:

                                                        self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: Processing {len(datafiles)} data files for LOB column {lob_col_name}")
                                                        current_datafile_num = 0

                                                        with concurrent.futures.ThreadPoolExecutor(max_workers=max_lob_parallel_workers) as executor:
                                                            futures = {}
                                                            for datafile_row in datafiles:
                                                                datafile = datafile_row[0]
                                                                occurrences = datafile_row[1]
                                                                current_datafile_num += 1

                                                                settings = {
                                                                    'target_schema_name': target_schema_name,
                                                                    'target_table_name': target_table_name,
                                                                    'primary_key_columns': migrator_tables.select_primary_key({'source_schema_name': table_data['source_schema_name'], 'source_table_name': table_data['source_table_name']}),
                                                                    'unl_import_table': table_name_for_lob_import,
                                                                    'lob_column': lob_col_name,
                                                                    'lob_col_index': lob_col_index,
                                                                    'lob_col_type': lob_col_type,
                                                                    'lob_columns_count': lob_columns_count,
                                                                    'target_columns': table_data['target_columns'],
                                                                    'datafile': datafile,
                                                                    'datafiles_count': len(datafiles),
                                                                    'current_datafile_num': current_datafile_num,
                                                                    'occurrences': occurrences,
                                                                    'lob_files_path': self.config_parser.get_source_data_export_file_path(),
                                                                }

                                                                self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: parallel LOB processing [{lob_col_name}]: futures running count: {len(futures)}")
                                                                while len(futures) >= max_lob_parallel_workers:

                                                                    done, _ = concurrent.futures.wait(futures, return_when=concurrent.futures.FIRST_COMPLETED)
                                                                    for future in done:
                                                                        datafile_done = futures.pop(future)
                                                                        if future.result() == False:
                                                                            if self.on_error_action == 'stop':
                                                                                self.config_parser.print_log_message('ERROR', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: parallel LOB processing [{lob_col_name}]: Stopping execution due to error.")
                                                                                exit(1)
                                                                        else:
                                                                            self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: parallel LOB processing [{lob_col_name}]: {datafile_done} LOBs migrated OK")

                                                                # Submit the next task
                                                                future = executor.submit(self.lob_worker, settings)
                                                                futures[future] = datafile

                                                            # Process remaining futures
                                                            self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: parallel LOB processing [{lob_col_name}]: Processing remaining futures")
                                                            for future in concurrent.futures.as_completed(futures):
                                                                datafile_done = futures[future]
                                                                if future.result() == False:
                                                                    if self.on_error_action == 'stop':
                                                                        self.config_parser.print_log_message('ERROR', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: parallel LOB processing [{lob_col_name}]: Stopping execution due to error.")
                                                                        exit(1)
                                                                else:
                                                                    self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: parallel LOB processing [{lob_col_name}]: {datafile_done} LOBs migrated OK")

                                                        self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: parallel LOB processing [{lob_col_name}]: All datafiles processed successfully.")
                                                    else:
                                                        self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: parallel LOB processing [{lob_col_name}]: No datafiles to process.")


                                                else:
                                                    self.config_parser.print_log_message('ERROR', f"orchestrator: table_worker: Worker {worker_id}: LOB column {lob_col_name} not found in target table {target_table_name}. Skipping LOB data migration for this column.")
                                                    continue


                                            self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Data copied successfully from CSV file {csv_file_name} to table {target_table_name} with LOB columns.")
                                            # Drop the intermediate import table if clean objects is True
                                            if clean_objects:
                                                self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Dropping intermediate import table {table_name_for_lob_import} as clean_objects is True.")
                                                worker_target_connection.execute_query(f'DROP TABLE IF EXISTS "{target_schema_name}"."{table_name_for_lob_import}" CASCADE')
                                                self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Intermediate import table {table_name_for_lob_import} dropped successfully.")
                                            else:
                                                self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Keeping intermediate import table {table_name_for_lob_import} as clean_objects is False.")

                                        else:

                                            if data_source_settings['lob_columns'] != '' and not self.config_parser.should_migrate_lob_values():
                                                self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Data source for table {target_table_name} has LOB columns: {data_source_settings['lob_columns']}, but LOB migration is disabled. Skipping LOB data migration.")

                                            # No LOB columns - standard CSV import
                                            # CSV data source - directly import into target database
                                            part_name = 'copy data from CSV'
                                            self.config_parser.print_log_message('DEBUG', f"orchestrator: table_worker: Worker {worker_id}: Executing COPY command: {copy_command}, csv_file_name: {csv_file_name}")
                                            worker_target_connection.copy_from_file(copy_command, csv_file_name)

                                            self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Data copied successfully from CSV file {csv_file_name} to table {target_table_name}.")

                                        target_table_rows = worker_target_connection.get_rows_count(target_schema_name, target_table_name)
                                        rows_migrated = target_table_rows
                                        if self.config_parser.get_source_db_type() == 'ibm_db2_zos':
                                            source_table_rows_all = target_table_rows
                                            source_table_rows_limited = target_table_rows
                                            migrator_tables.update_data_migration_rows({
                                                "row_id": protocol_id,
                                                "source_table_rows_all": source_table_rows_all,
                                                "source_table_rows_limited": source_table_rows_limited,
                                                "target_table_rows": target_table_rows,
                                            })
                                            migrator_tables.update_table_rows_counts({
                                                "row_id": table_data['id'],
                                                "source_table_rows_all": source_table_rows_all,
                                                "source_table_rows_limited": source_table_rows_limited,
                                                "target_table_rows": target_table_rows,
                                            })
                                        data_import_end_time = time.time()
                                        data_import_duration = data_import_end_time - data_import_start_time
                                        self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Table {target_table_name}: Data import duration: {data_import_duration:.2f} seconds, rows migrated: {target_table_rows}.")

                                        # self.migrator_tables.update_table_status(table_data['id'], True, f'migrated OK ({rows_migrated} rows)')
                                        migrator_tables.update_data_migration_status({
                                            'row_id': protocol_id,
                                            'success': True,
                                            'message': 'OK',
                                            'target_table_rows': target_table_rows,
                                            'batch_count': 0,
                                            'shortest_batch_seconds': data_import_duration,
                                            'longest_batch_seconds': data_import_duration,
                                            'average_batch_seconds': data_import_duration,
                                        })

                                        if data_source_settings['format_options']['format'].upper() == 'UNL' and clean_objects:
                                            # Delete the converted CSV file after import if clean_objects is True
                                            try:
                                                if csv_file_name and os.path.exists(csv_file_name):
                                                    os.remove(csv_file_name)
                                                    self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Deleted temporary CSV file {csv_file_name}.")
                                            except Exception as cleanup_exc:
                                                self.config_parser.print_log_message('ERROR', f"orchestrator: table_worker: Worker {worker_id}: Failed to delete temporary CSV file {csv_file_name}: {cleanup_exc}")

                                    except Exception as e:
                                        self.migrator_tables.update_table_status({'row_id': table_data['id'], 'success': False, 'message': f'ERROR: {e}'})
                                        self.config_parser.print_log_message('ERROR', f"orchestrator: table_worker: Worker {worker_id}: ({part_name}) Error copying data from CSV file {csv_file_name} to table {target_table_name}: {e}")
                                        return False
                    else:
                        if not data_source['file_found'] and data_source['file_name'] is not None:
                            self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: ({part_name}) Data file {data_source['file_name']} not found for table {target_table_name}.")

                            on_missing_data_file = self.config_parser.get_source_data_export_on_missing_data_file()
                            if on_missing_data_file == 'skip':
                                self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Skipping data migration for table {target_table_name} due to missing data file.")
                                use_source_table = False
                                rows_migrated = 0
                            else:
                                self.config_parser.print_log_message('ERROR', f"orchestrator: table_worker: Worker {worker_id}: Data file {data_source['file_name']} not found for table {target_table_name}. Switching to source table.")
                                use_source_table = True

                else:
                    use_source_table = True

                if use_source_table:
                    part_name = 'migrate data'
                    self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Migrating data for table {target_table_name} from source database.")

                    table_settings = {
                        'worker_id': worker_id,
                        'source_schema_name': table_data['source_schema_name'],
                        'source_table_name': table_data['source_table_name'],
                        'source_table_id': table_data['source_table_id'],
                        'source_columns': table_data['source_columns'],
                        'target_schema_name': target_schema_name,
                        'target_table_name': target_table_name,
                        'target_columns': table_data['target_columns'],
                        'table_comment': table_data['table_comment'],
                        # 'primary_key_columns': table_data['primary_key_columns'],
                        # 'primary_key_columns_count': table_data['primary_key_columns_count'],
                        # 'primary_key_columns_types': table_data['primary_key_columns_types'],
                        'batch_size': self.config_parser.get_table_batch_size(table_data['source_table_name']),
                        'migrator_tables': settings['migrator_tables'],
                        'migration_limitation': '',
                        'chunk_size': self.config_parser.get_table_chunk_size(table_data['source_table_name']),
                        'chunk_number': 1,
                        'resume_after_crash': settings['resume_after_crash'],
                        'drop_unfinished_tables': settings['drop_unfinished_tables'],
                        'source_table_rows_all': table_data.get('source_table_rows_all', 0)
                    }

                    rows_migration_limitations = settings['migrator_tables'].get_records_data_migration_limitation(table_data['source_table_name'])
                    migration_limitations = []
                    if rows_migration_limitations:
                        self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Found data migration limitations matching table {target_table_name}: {rows_migration_limitations}")
                        for limitation in rows_migration_limitations:
                            where_clause = limitation[0]
                            where_clause = where_clause.replace('{source_schema_name}', table_data['source_schema_name']).replace('{source_table_name}', table_data['source_table_name'])
                            use_when_column_name = limitation[1]
                            for col_order_num, column_info in table_data['source_columns'].items():
                                column_name = column_info['column_name']
                                if column_name == use_when_column_name or re.match(use_when_column_name, column_name):
                                    self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Column {column_name} matches migration limitation.")
                                    migration_limitations.append(where_clause)
                        if migration_limitations:
                            table_settings['migration_limitation'] = f"{' AND '.join(migration_limitations)}" if migration_limitations else ''
                            self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Migration limitations for table {target_table_name}: {migration_limitations}")

                    while True:
                        if self.config_parser.pause_migration_fired():
                            self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Migration paused for table {target_table_name}.")
                            self.config_parser.wait_for_resume()
                            self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Resuming migration for table {target_table_name}.")
                        # proper pausing / stopping of the migration requires to connect and disconnect for each chunk
                        migration_stats = worker_source_connection.migrate_table(worker_target_connection, table_settings)

                        rows_migrated += migration_stats['rows_migrated']
                        if migration_stats['finished']:
                            break
                        table_settings['chunk_number'] += 1

                next_identity = worker_source_connection.get_table_next_identity(table_data['source_schema_name'], table_data['source_table_name'])
                worker_source_connection.disconnect()

                if rows_migrated > 0:
                    # sequences setting
                    part_name = 'sequences'
                    self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Setting sequences for table {target_table_name} in target database.")
                    sequences = worker_target_connection.fetch_table_sequences(target_schema_name, target_table_name)
                    if sequences:
                        for order_num, sequence_details in sequences.items():
                            sequence_id = sequence_details['id']
                            sequence_name = sequence_details['name']
                            column_name = sequence_details['column_name']
                            sequence_sql = sequence_details['set_sequence_sql']
                            if next_identity is not None:
                                sequence_sql = f"SELECT setval('\"{target_schema_name}\".\"{sequence_name}\"', {next_identity}, false);"

                            self.migrator_tables.insert_sequence({
                                'sequence_id': sequence_id,
                                'target_schema_name': target_schema_name,
                                'target_table_name': target_table_name,
                                'target_column_name': column_name,
                                'target_sequence_name': sequence_name,
                                'target_sequence_sql': sequence_sql
                            })
                            self.config_parser.print_log_message( 'DEBUG', f"orchestrator: table_worker: Worker {worker_id}: Setting sequence with SQL: {sequence_sql}")
                            try:
                                worker_target_connection.execute_query(sequence_sql)
                                self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Sequence ({order_num}) {sequence_name} set successfully for table {target_table_name}.")
                                seq_curr_val = worker_target_connection.get_sequence_current_value(sequence_id)
                                self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Current value of sequence {sequence_name} is {seq_curr_val}.")
                                self.migrator_tables.update_sequence_status({'sequence_id': sequence_id, 'success': True, 'message': 'migrated OK'})
                            except Exception as e:
                                self.migrator_tables.update_sequence_status({'sequence_id': sequence_id, 'success': False, 'message': f'ERROR: {e}'})
                                self.migrator_tables.update_table_status({'row_id': table_data['id'], 'success': False, 'message': f'ERROR: {e}'})
                                self.config_parser.print_log_message('ERROR', f"orchestrator: table_worker: Worker {worker_id}: Error setting sequence {sequence_name} for table {target_table_name}: {e}")
                    else:
                        self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: No sequences found for table {target_table_name}.")
                else:
                    self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: No data found for table {target_table_name} - skipping sequences.")
            else:
                self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Skipping data migration for table {target_table_name} based on configuration")

            worker_end_time = time.time()
            elapsed_time = worker_end_time - worker_start_time
            self.config_parser.print_log_message('INFO', f"orchestrator: table_worker: Worker {worker_id}: Total time spent on migrating table {target_table_name} is {elapsed_time:.2f} seconds ({elapsed_time / 60:.2f} minutes)")
            try:
                worker_target_connection.disconnect()
            except Exception as e:
                pass
            return True
        except Exception as e_main:
            try:
                worker_source_connection.disconnect()
            except Exception as e:
                pass
            try:
                worker_target_connection.disconnect()
            except Exception as e:
                pass
            self.migrator_tables.update_table_status({'row_id': table_data['id'], 'success': False, 'message': f'ERROR: {e_main}'})
            self.handle_error(e_main, f"table_worker {worker_id} ({part_name}) {target_table_name}")
            return False

    def lob_worker(self, settings):
        part_name = 'start of lob_worker'
        target_schema_name = settings['target_schema_name']
        target_table_name = settings['target_table_name']
        primary_key_columns = settings['primary_key_columns']
        unl_import_table = settings['unl_import_table']
        all_lob_columns = settings.get('all_lob_columns')
        lob_column = settings['lob_column']
        lob_col_index = settings['lob_col_index']
        lob_col_type = settings['lob_col_type']
        lob_columns_count = settings.get('lob_columns_count', 1)
        target_columns = settings['target_columns']
        datafile = settings['datafile']
        datafiles_count = settings['datafiles_count']
        current_datafile_num = settings['current_datafile_num']
        occurrences = settings['occurrences']
        lob_files_path = settings['lob_files_path']

        try:
            worker_insert_connection = self.load_connector('target')
            worker_insert_connection.connect()
            worker_insert_connection.autocommit = True  # Enable autocommit for the insert connection

            processing_start_time = time.time()
            worker_id = uuid.uuid4()
            self.config_parser.print_log_message('INFO', f"orchestrator: lob_worker: Worker {worker_id}: started for LOB file: {datafile}, LOB column {lob_column} - occurrences: {occurrences} - {current_datafile_num}/{datafiles_count}")

            part_name = 'prepare insert SQL'
            col_list = ', '.join([f'"{col_info["column_name"]}"' for _, col_info in target_columns.items()])
            placeholders = ', '.join(['%s'] * len(target_columns))

            basic_insert_sql = f'''INSERT INTO "{target_schema_name}"."{target_table_name}" ({col_list}) VALUES ({placeholders})'''

            if primary_key_columns is None or primary_key_columns.strip() == '' or lob_columns_count <= 1:

                part_name = 'build merge statement no primary key'
                # No primary key defined - build MERGE statement using all non-LOB columns as match conditions

                # Build match conditions from all columns excluding LOB columns
                all_lob_columns = settings.get('all_lob_columns', [])
                if all_lob_columns is None:
                    all_lob_columns = []

                # Get all column names excluding LOB columns
                non_lob_columns = []
                for _, col_info in target_columns.items():
                    col_name = col_info['column_name'].strip().strip('"')
                    if col_name not in all_lob_columns:
                        non_lob_columns.append(col_name)

                # Build match conditions from non-LOB columns
                if non_lob_columns:
                    merge_match_conditions = ' AND '.join([f'target."{col}" = source."{col}"' for col in non_lob_columns])
                # else:
                #     # Fallback to primary key if no non-LOB columns available
                #     if primary_key_columns and primary_key_columns.strip():
                #         pk_columns_list = [col.strip().strip('"') for col in primary_key_columns.split(',')]
                #         merge_match_conditions = ' AND '.join([f'target."{pk_col}" = source."{pk_col}"' for pk_col in pk_columns_list])
                #     else:
                #         # If no primary key and no non-LOB columns, use all columns as fallback
                #         merge_match_conditions = ' AND '.join([f'target."{col_info["column_name"].strip().strip('"')}" = source."{col_info["column_name"].strip().strip('"')}"' for _, col_info in target_columns.items()])

                # Build the column assignments for INSERT and UPDATE
                insert_columns = ', '.join([f'"{col_info["column_name"]}"' for _, col_info in target_columns.items()])
                insert_values = ', '.join([f'source."{col_info["column_name"]}"' for _, col_info in target_columns.items()])

                # Build cast expressions for each column based on its data type
                cast_expressions = []
                for _, col_info in target_columns.items():
                    col_name = col_info["column_name"]
                    data_type = col_info["data_type"]
                    if data_type.lower() == 'oid':
                        cast_expressions.append(f'lo_from_bytea(0, %s::bytea) AS "{col_name}"')
                    else:
                        cast_expressions.append(f'%s::{data_type} AS "{col_name}"')

                lob_val_expr = 'lo_from_bytea(0, %s::bytea)' if lob_col_type.lower() == 'oid' else f'%s::{lob_col_type}'

                repeat_insert_sql = f'''
                    WITH source_data AS (
                        SELECT {', '.join(cast_expressions)}
                    )
                    MERGE INTO "{target_schema_name}"."{target_table_name}" AS target
                    USING source_data AS source
                    ON {merge_match_conditions}
                    WHEN MATCHED THEN
                        UPDATE SET {lob_column} = {lob_val_expr}
                    WHEN NOT MATCHED THEN
                        INSERT ({insert_columns})
                        VALUES ({insert_values})
                '''

            else:

                part_name = 'build insert statement with primary key'
                # we have primary key defined - use it for INSERT ... ON CONFLICT
                lob_val_expr = 'lo_from_bytea(0, %s::bytea)' if lob_col_type.lower() == 'oid' else f'%s::{lob_col_type}'
                repeat_insert_sql = f'''INSERT INTO "{target_schema_name}"."{target_table_name}" ({col_list}) VALUES ({placeholders}) ON CONFLICT ({primary_key_columns}) DO UPDATE SET {lob_column} = {lob_val_expr}'''

                # pk_columns_list = [col.strip().strip('"') for col in primary_key_columns.split(',')]
                # merge_match_conditions = ' AND '.join([f'target."{pk_col}" = source."{pk_col}"' for pk_col in pk_columns_list])


            worker_select_connection = self.load_connector('target')
            worker_select_connection.connect()
            # cur_select = worker_select_connection.connection.cursor()

            select_cur = worker_select_connection.connection.cursor()
            select_sql = f"""SELECT {col_list} FROM "{target_schema_name}"."{unl_import_table}" WHERE coalesce(split_part({lob_column},',',3), '0,0,0') = '{datafile}'"""

            self.config_parser.print_log_message('DEBUG', f"orchestrator: lob_worker: Worker {worker_id}: LOB file: {datafile}, LOB column {lob_column}: Fetching rows from '{unl_import_table}' with query: {select_sql}")
            self.config_parser.print_log_message('DEBUG', f"orchestrator: lob_worker: Worker {worker_id}: LOB file: {datafile}, LOB column {lob_column}: Basic Insert SQL: {basic_insert_sql}")
            self.config_parser.print_log_message('DEBUG', f"orchestrator: lob_worker: Worker {worker_id}: LOB file: {datafile}, LOB column {lob_column}: Repeat Insert SQL: {repeat_insert_sql}")
            self.config_parser.print_log_message('DEBUG', f"orchestrator: lob_worker: Worker {worker_id}: LOB file: {datafile}, LOB column: {lob_column} (index: {lob_col_index}, type: {lob_col_type})")
            counter = 0
            read_lines = 0

            select_cur.execute(select_sql)
            row = select_cur.fetchone()
            part_name = 'LOB processing start'
            while row is not None:
                row = list(row)
                read_lines += 1

                part_name = 'read lob pointer'
                if lob_col_index <= 0 or lob_col_index > len(row):
                    self.config_parser.print_log_message('ERROR', f"orchestrator: lob_worker: Worker: {worker_id}: LOB file: {datafile}, LOB column: {lob_column}: Invalid lob_col_index {lob_col_index} for row of length {len(row)}. Skipping row.")
                    row = select_cur.fetchone()
                    continue
                content = row[lob_col_index-1]
                content_is_null = False

                part_name = 'decode LOB pointer'
                if content is not None and content != '' and content != '0,0,0':
                    parts = content.split(',')
                    start = int(parts[0], 16)
                    length = int(parts[1], 16)
                    datafile = parts[2]
                    filepath = os.path.join(lob_files_path, datafile)
                else:
                    content_is_null = True
                    start = 0
                    length = 0
                    filepath = None

                # if length > 1024*1024*1024:
                #   self.config_parser.print_log_message('INFO', f"orchestrator: lob_worker: Worker: {worker_id}: LOB size exceeds 1GB: {length} bytes ({length / 1024 / 1024 / 1024}) GB - in row: {row} - setting to NULL.")
                #   content_is_null = True
                # if args.test_sizes:
                #   row = select_cur.fetchone() # Fetch in test mode
                #   continue

                if not content_is_null and os.path.exists(filepath):

                    try:
                        part_name = 'read LOB file'
                        if lob_col_type.lower() == 'bytea':
                            with open(filepath, 'rb') as f:
                                f.seek(start)
                                chunk = f.read(length)

                        elif lob_col_type.lower() == 'text':
                            with open(filepath, 'rb') as f:
                                f.seek(start)
                                chunk = f.read(length).decode('utf-8')

                        row[lob_col_index-1] = chunk
                    except Exception as e:
                        self.config_parser.print_log_message('ERROR', f"orchestrator: lob_worker: Worker: {worker_id}: Error reading LOB file: {filepath}, LOB column: {lob_column}: (start: {start}, length: {length}) in row: {row} - setting LOB value to NULL. Error: {e}")
                        content_is_null = True

                elif not content_is_null and not os.path.exists(filepath):
                    self.config_parser.print_log_message('ERROR', f"orchestrator: lob_worker: Worker: {worker_id}: LOB file not found: {filepath}, LOB column: {lob_column}: - in row: {row} - setting LOB value to NULL.")
                    content_is_null = True

                if content_is_null:
                    row[lob_col_index-1] = None

                try:
                    part_name = 'insert row basic'
                    cur_insert = worker_insert_connection.connection.cursor()
                    try:
                        cur_insert.execute(basic_insert_sql, row + [row[lob_col_index-1]])
                        worker_insert_connection.connection.commit()
                    except Exception as e:
                        try:
                            part_name = 'insert row repeat'
                            cur_insert.execute(repeat_insert_sql, row + [row[lob_col_index-1]])
                            worker_insert_connection.connection.commit()
                        except Exception as e:
                            worker_insert_connection.connection.rollback()
                            raise e
                    cur_insert.close()
                    counter += 1
                except Exception as e:
                    cur_insert.close()
                    self.config_parser.print_log_message('ERROR', f"orchestrator: lob_worker: Worker: {worker_id} [{part_name}]: LOB file: {datafile}, LOB column {lob_column}: Error executing INSERT command for row: {row} (counter: {counter}), error: {e}")

                part_name = 'fetch next row'
                row = select_cur.fetchone()
            select_cur.close()
            self.config_parser.print_log_message('INFO', f"orchestrator: lob_worker: Worker: {worker_id}: Processed {read_lines} rows from '{unl_import_table}', datafile: {datafile}, LOB column {lob_column}: occurrences: {occurrences} - inserted {counter} into '{target_schema_name}.{target_table_name}'.")

            worker_select_connection.disconnect()
            worker_insert_connection.disconnect()

            self.config_parser.print_log_message('INFO', f"orchestrator: lob_worker: Worker: {worker_id}: Processing completed in {time.time() - processing_start_time} seconds.")
            return True  # Indicate successful processing of this datafile
        except Exception as e:
            self.handle_error(e, f"lob_worker: Worker: {worker_id}: Datafile: {datafile}: Table: {target_table_name}: LOB file: {datafile}, LOB column {lob_column}: {part_name} - {e}")
            try:
                worker_select_connection.disconnect()
            except Exception as e:
                pass
            try:
                worker_insert_connection.disconnect()
            except Exception as e:
                pass
            return False  # Indicate failure in processing this datafile

    def index_worker(self, index_data, target_db_type):
        worker_id = uuid.uuid4()
        try:
            index_name = index_data['index_name']
            create_index_sql = index_data['index_sql']

            self.migrator_tables.update_protocol_task_started('indexes', index_data['id'])
            self.config_parser.print_log_message('INFO', f"orchestrator: index_worker: Worker {worker_id}: Creating index {index_name} in target database.")

            # Each worker uses its own separate connection to the target database
            worker_target_connection = self.load_connector('target')

            self.config_parser.print_log_message( 'DEBUG', f"orchestrator: index_worker: Worker {worker_id}: Creating index with SQL: {create_index_sql}")

            worker_target_connection.connect()

            if worker_target_connection.session_settings:
                self.config_parser.print_log_message( 'DEBUG', f"orchestrator: index_worker: Worker {worker_id}: Executing session settings: {worker_target_connection.session_settings}")
                worker_target_connection.execute_query(worker_target_connection.session_settings)

            worker_target_connection.execute_query(create_index_sql)
            self.config_parser.print_log_message('INFO', f"orchestrator: index_worker: Worker {worker_id}: Index '{index_name}' created successfully.")

            worker_target_connection.disconnect()
            return True
        except Exception as e:
            self.migrator_tables.update_index_status({'row_id': index_data['id'], 'success': False, 'message': f'ERROR: {e}'})
            self.handle_error(e, f"index_worker {worker_id} {index_name}")
            return False

    def constraint_worker(self, constraint_data, target_db_type):
        worker_id = uuid.uuid4()
        try:
            constraint_name = constraint_data['constraint_name']
            self.migrator_tables.update_protocol_task_started('constraints', constraint_data['id'])
            self.config_parser.print_log_message('INFO', f"orchestrator: constraint_worker: Worker {worker_id}: Creating constraint {constraint_name} in target database.")
            create_constraint_sql = constraint_data['constraint_sql']

            target_schema_name = constraint_data['target_schema_name']
            target_table_name = constraint_data['target_table_name']
            # referenced_table_schema = constraint_data['target_schema_name']
            referenced_table_schema = constraint_data['referenced_table_schema']
            referenced_table_name = constraint_data['referenced_table_name']

            if create_constraint_sql:

                # Each worker uses its own separate connection to the target database
                worker_target_connection = self.load_connector('target')
                worker_target_connection.connect()

                use_aliases_as_target_names = self.config_parser.get_use_aliases_as_target_names()

                target_table_name_to_check = constraint_data['target_alias_name'] if use_aliases_as_target_names and constraint_data.get('target_alias_name') else target_table_name
                target_table_name_to_check = self.config_parser.convert_names_case(target_table_name_to_check)

                if not worker_target_connection.target_table_exists(target_schema_name, target_table_name_to_check):
                    self.config_parser.print_log_message('ERROR', f"orchestrator: constraint_worker: Worker {worker_id}: Target table {target_schema_name}.{target_table_name_to_check} for constraint {constraint_name} does not exist - skipping constraint creation.")
                    self.migrator_tables.update_constraint_status({'row_id': constraint_data['id'], 'success': False, 'message': f'ERROR: target table {target_schema_name}.{target_table_name_to_check} does not exist'})
                    return False

                # if constraint_data['constraint_type'] is 'FOREIGN KEY': ## original version on main, must be tested
                if referenced_table_name and referenced_table_name.strip():
                    referenced_target_table = self.migrator_tables.select_table_by_source({'source_schema_name': referenced_table_schema, 'source_table_name': referenced_table_name})
                    if referenced_target_table is None:
                        self.config_parser.print_log_message('INFO', f"orchestrator: type is {constraint_data['constraint_type']}")
                        self.config_parser.print_log_message('ERROR', f"orchestrator: constraint_worker: Worker {worker_id}: Referenced table {referenced_table_schema}.{referenced_table_name} for constraint {constraint_name} not found - skipping constraint creation.")
                        self.migrator_tables.update_constraint_status({'row_id': constraint_data['id'], 'success': False, 'message': f'ERROR: referenced table {referenced_table_schema}.{referenced_table_name} not found'})
                        worker_target_connection.disconnect()
                        return False

                    referenced_target_table_name_to_check = referenced_target_table['target_alias_name'] if use_aliases_as_target_names and referenced_target_table.get('target_alias_name') else referenced_target_table['target_table_name']
                    referenced_target_table_name_to_check = self.config_parser.convert_names_case(referenced_target_table_name_to_check)

                    if not worker_target_connection.target_table_exists(referenced_target_table['target_schema_name'], referenced_target_table_name_to_check):
                        self.config_parser.print_log_message('ERROR', f"orchestrator: constraint_worker: Worker {worker_id}: Referenced table {referenced_target_table['target_schema_name']}.{referenced_target_table_name_to_check} for constraint {constraint_name} does not exist - skipping constraint creation.")
                        self.migrator_tables.update_constraint_status({'row_id': constraint_data['id'], 'success': False, 'message': f"ERROR: referenced table {referenced_target_table['target_schema_name']}.{referenced_target_table_name_to_check} does not exist"})
                        return False


                self.config_parser.print_log_message( 'DEBUG', f"orchestrator: constraint_worker: Worker {worker_id}: Creating constraint with SQL: {create_constraint_sql}")

                query = f'''SET SESSION search_path TO {constraint_data['target_schema_name']};'''

                worker_target_connection.execute_query(query)

                if worker_target_connection.session_settings:
                    self.config_parser.print_log_message( 'DEBUG', f"orchestrator: constraint_worker: Worker {worker_id}: Executing session settings: {worker_target_connection.session_settings}")
                    worker_target_connection.execute_query(worker_target_connection.session_settings)

                creation_try = 0
                while True:
                    try:
                        worker_target_connection.execute_query(create_constraint_sql)
                        break  # Exit loop if successful
                    except Exception as e:
                        self.config_parser.print_log_message('ERROR', f"orchestrator: constraint_worker: Worker {worker_id}: Error creating constraint {constraint_name}: {e}")
                        error_texts = [
                            "no unique constraint matching",
                            "is violated by some row",
                            "violates foreign key constraint",
                            "does not exist",
                            "violates check constraint",
                        ]
                        if any(txt in str(e).lower() for txt in error_texts):
                            self.migrator_tables.update_constraint_status({'row_id': constraint_data['id'], 'success': False, 'message': f'ERROR: {e}'})
                            worker_target_connection.disconnect()
                            return False
                        self.config_parser.print_log_message('INFO', f"orchestrator: constraint_worker: Worker {worker_id}: Retrying ({creation_try}) to create constraint {constraint_name}...")
                        if creation_try > 5:
                            self.handle_error(e, f"constraint_worker {worker_id}: Failed to create {constraint_name} after {creation_try} attempts.")
                            self.migrator_tables.update_constraint_status({'row_id': constraint_data['id'], 'success': False, 'message': f'ERROR: {e}'})
                            worker_target_connection.disconnect()
                            return False
                        creation_try += 1
                        time.sleep(5)  # Wait before retrying

                query = 'RESET search_path;'
                worker_target_connection.execute_query(query)
                self.config_parser.print_log_message('INFO', f"orchestrator: constraint_worker: Worker {worker_id}: Constraint '{constraint_name}' created successfully.")
                worker_target_connection.disconnect()
                return True
            else:
                self.config_parser.print_log_message('INFO', f"orchestrator: constraint_worker: Worker {worker_id}: Constraint {constraint_name} does not have a SQL statement - skipping.")
                worker_target_connection.disconnect()
                return False

        except Exception as e:
            self.handle_error(e, f"constraint_worker {worker_id} {constraint_name}")
            return False

    def stdwf_migrate_funcprocs(self):
        self.migrator_tables.insert_main({'task_name': 'Orchestrator', 'subtask_name': 'functions/procedures migration'})
        include_funcprocs = self.config_parser.get_include_funcprocs()
        exclude_funcprocs = self.config_parser.get_exclude_funcprocs() or []

        if self.config_parser.should_migrate_funcprocs():
            self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_funcprocs: Migrating functions and procedures.")
            funcproc_names = self.source_connection.fetch_funcproc_names(self.config_parser.get_source_schema())
            self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_funcprocs: Function/procedure names: {funcproc_names}")

            if funcproc_names:
                for order_num, funcproc_data in funcproc_names.items():
                    self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_funcprocs: Processing func/proc {order_num}/{len(funcproc_names)}: {funcproc_data['name']}")
                    if include_funcprocs == ['.*'] or '.*' in include_funcprocs:
                        pass
                    elif not any(fnmatch.fnmatch(funcproc_data['name'], pattern) for pattern in include_funcprocs):
                        continue
                    if any(fnmatch.fnmatch(funcproc_data['name'], pattern) for pattern in exclude_funcprocs):
                        self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_funcprocs: Func/proc {funcproc_data['name']} is excluded from migration.")
                        continue

                    funcproc_id = funcproc_data['id']
                    funcproc_type = funcproc_data['type']
                    self.migrator_tables.update_protocol_task_started('funcprocs', funcproc_data['id'])
                    self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_funcprocs: Migrating {funcproc_type} {funcproc_data['name']}.")
                    try:
                        funcproc_code = self.source_connection.fetch_funcproc_code(funcproc_id)

                        # Handle dict return (with implicit schema) for logging/DB
                        funcproc_code_str = funcproc_code
                        if isinstance(funcproc_code, dict):
                            funcproc_code_str = funcproc_code.get('definition', '') or ''
                        else:
                            funcproc_code_str = str(funcproc_code) if funcproc_code else ''

                        table_names = []
                        view_names = []
                        converted_code = ''
                        try:
                            table_names = self.migrator_tables.fetch_all_target_table_names()
                        except Exception as e:
                            self.handle_error(e, 'fetching table names')
                        try:
                            view_names = self.migrator_tables.fetch_all_target_view_names()
                        except Exception as e:
                            self.handle_error(e, 'fetching view names')

                        self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_funcprocs: Converting {funcproc_type} {funcproc_data['name']} code...")
                        converted_code = self.source_connection.convert_funcproc_code({
                            'funcproc_code': funcproc_code,
                            'funcproc_name': funcproc_data['name'],
                            'target_db_type': self.config_parser.get_target_db_type(),
                            'source_schema_name': self.config_parser.get_source_schema(),
                            'target_schema_name': self.config_parser.get_target_schema(),
                            'table_list': table_names,
                            'view_list': view_names,
                            'migrator_tables': self.migrator_tables,
                            })

                        self.config_parser.print_log_message( 'DEBUG', "orchestrator: run_migrate_funcprocs: Checking for remote objects substitution in functions/procedures...")
                        rows = self.migrator_tables.get_records_remote_objects_substitution()
                        if rows:
                            for row in rows:
                                self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_funcprocs: Funcs/Procs - remote objects substituting {row[0]} with {row[1]}")
                                converted_code = re.sub(re.escape(row[0]), row[1], converted_code, flags=re.IGNORECASE | re.MULTILINE | re.DOTALL)

                        self.migrator_tables.insert_funcprocs({
                            'source_schema_name': self.source_schema_name,
                            'source_funcproc_name': funcproc_data['name'],
                            'source_funcproc_id': funcproc_id,
                            'source_funcproc_sql': funcproc_code_str,
                            'target_schema_name': self.target_schema_name,
                            'target_funcproc_name': funcproc_data['name'],
                            'target_funcproc_sql': converted_code,
                            'funcproc_comment': funcproc_data['comment']
                        })

                        if converted_code is not None and converted_code.strip():
                            self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_funcprocs: Creating {funcproc_type} {funcproc_data['name']} in target database.")
                            self.target_connection.connect()

                            if self.target_connection.session_settings:
                                self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_funcprocs: Executing session settings: {self.target_connection.session_settings}")
                                self.target_connection.execute_query(self.target_connection.session_settings)

                            self.target_connection.execute_query(converted_code)
                            self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_funcprocs: [OK] Source code for {funcproc_data['name']}: {funcproc_code_str}")
                            self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_funcprocs: [OK] Converted code for {funcproc_data['name']}: {converted_code}")
                            self.migrator_tables.update_funcproc_status({'source_funcproc_id': funcproc_id, 'success': True, 'message': 'migrated OK'})
                        else:
                            self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_funcprocs: Skipping {funcproc_type} {funcproc_data['name']} - no conversion done")
                            self.migrator_tables.update_funcproc_status({'source_funcproc_id': funcproc_id, 'success': False, 'message': 'no conversion'})
                        self.target_connection.disconnect()
                    except Exception as e:
                        self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_funcprocs: [ERROR] Source code for {funcproc_data['name']}: {funcproc_code}")
                        self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_funcprocs: [ERROR] Converted code for {funcproc_data['name']}: {converted_code}")
                        self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_funcprocs: [ERROR] Migrating {funcproc_type} {funcproc_data['name']}.")
                        self.migrator_tables.update_funcproc_status({'source_funcproc_id': funcproc_id, 'success': False, 'message': f'ERROR: {e}'})
                        self.config_parser.print_log_message('ERROR', f"orchestrator: run_migrate_funcprocs: Error migrating {funcproc_type} {funcproc_data['name']}: {e}")
                        self.config_parser.print_log_message('ERROR', traceback.format_exc())
                        try:
                            self.target_connection.disconnect()
                        except:
                            pass

                self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_funcprocs: Functions and procedures migrated successfully.")
            else:
                self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_funcprocs: No functions or procedures found to migrate.")
        else:
            self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_funcprocs: Skipping function and procedure migration as requested.")

        self.migrator_tables.update_main_status({'task_name': 'Orchestrator', 'subtask_name': 'functions/procedures migration', 'success': True, 'message': 'finished OK'})

    def stdwf_migrate_triggers(self):
        self.migrator_tables.insert_main({'task_name': 'Orchestrator', 'subtask_name': 'triggers migration'})
        try:
            if self.config_parser.should_migrate_triggers():
                self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_triggers: Migrating triggers.")

                all_triggers = self.migrator_tables.fetch_all_triggers()
                if all_triggers:
                    for one_trigger in all_triggers:
                        trigger_detail = self.migrator_tables.decode_trigger_row(one_trigger)

                        if self.config_parser.should_migrate_triggers(trigger_detail['source_table_name']):
                            self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_triggers: Processing trigger {trigger_detail['trigger_name']}")
                            self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_triggers: Trigger details: {trigger_detail}")

                            converted_code = trigger_detail['trigger_target_sql']

                            self.config_parser.print_log_message( 'DEBUG', "orchestrator: run_migrate_triggers: Checking for remote objects substitution in triggers...")
                            rows = self.migrator_tables.get_records_remote_objects_substitution()
                            if rows:
                                for row in rows:
                                    self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_triggers: Triggers - remote objects substituting {row[0]} with {row[1]}")
                                    converted_code = re.sub(re.escape(row[0]), row[1], converted_code, flags=re.IGNORECASE | re.MULTILINE | re.DOTALL)

                            try:
                                if converted_code is not None and converted_code.strip():
                                    self.migrator_tables.update_protocol_task_started('triggers', trigger_detail['id'])
                                    self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_triggers: Creating trigger {trigger_detail['trigger_name']} in target database.")
                                    self.target_connection.connect()
                                    self.target_connection.execute_query(converted_code)
                                    self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_triggers: [OK] Source code for {trigger_detail['trigger_name']}: {trigger_detail['trigger_source_sql']}")
                                    self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_triggers: [OK] Converted code for {trigger_detail['trigger_name']}: {converted_code}")
                                    self.migrator_tables.update_trigger_status({'row_id': trigger_detail['id'], 'success': True, 'message': 'migrated OK'})
                                else:
                                    self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_triggers: Skipping trigger {trigger_detail['trigger_name']} - no conversion.")
                                    self.migrator_tables.update_trigger_status({'row_id': trigger_detail['id'], 'success': False, 'message': 'no conversion'})
                                self.target_connection.disconnect()
                            except Exception as e:
                                self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_triggers: [ERROR] Migrating trigger {trigger_detail['trigger_name']}.")
                                self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_triggers: [ERROR] Source code for {trigger_detail['trigger_name']}: {trigger_detail['trigger_source_sql']}")
                                self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_triggers: [ERROR] Converted code for {trigger_detail['trigger_name']}: {converted_code}")
                                # self.migrator_tables.update_trigger_status(trigger_detail['id'], False, f'ERROR: {e}')
                                self.migrator_tables.update_trigger_status({'row_id': trigger_detail['id'], 'success': False, 'message': f'ERROR: {e}'})
                                # self.handle_error(e, f"migrate_trigger {trigger_detail['trigger_name']}")
                                # We do not want to stop the whole migration if one trigger fails
                                self.config_parser.print_log_message('ERROR', f"orchestrator: run_migrate_triggers: Error migrating trigger {trigger_detail['trigger_name']}: {e}")
                                self.config_parser.print_log_message('ERROR', traceback.format_exc())
                        else:
                            self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_triggers: Skipping trigger {trigger_detail['trigger_name']} for table {trigger_detail['table_name']} based on the migration configuration.")

                    self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_triggers: Triggers migrated successfully.")
                else:
                    self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_triggers: No triggers found to migrate.")
            else:
                self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_triggers: Skipping trigger migration as requested.")

            self.migrator_tables.update_main_status({'task_name': 'Orchestrator', 'subtask_name': 'triggers migration', 'success': True, 'message': 'finished OK'})

        except Exception as e:
            self.handle_error(e, 'migrate_triggers')

    def view_worker(self, view_detail):
        worker_id = uuid.uuid4()
        view_type_str = "alias view" if view_detail.get('alias_view', False) else "normal view"
        try:
            # Each worker uses its own separate connection to the target database
            worker_target_connection = self.load_connector('target')
            worker_target_connection.connect()

            if worker_target_connection.session_settings:
                self.config_parser.print_log_message( 'DEBUG', f"orchestrator: view_worker: Worker {worker_id}: Executing session settings for {view_type_str}: {worker_target_connection.session_settings}")
                worker_target_connection.execute_query(worker_target_connection.session_settings)

            query = f'''SET SESSION search_path TO {view_detail['target_schema_name']};'''
            worker_target_connection.execute_query(query)

            self.migrator_tables.update_protocol_task_started('views', view_detail['id'])
            self.config_parser.print_log_message( 'DEBUG', f"orchestrator: view_worker: Worker {worker_id}: Creating {view_type_str} {view_detail['source_view_name']} with SQL: {view_detail['target_view_sql']}")
            worker_target_connection.execute_query(view_detail['target_view_sql'])

            # Commit the view creation
            if hasattr(worker_target_connection, 'connection') and hasattr(worker_target_connection.connection, 'commit'):
                worker_target_connection.connection.commit()

            query = f'''RESET search_path;'''
            worker_target_connection.execute_query(query)
            worker_target_connection.disconnect()
            return True, "migrated OK"
        except Exception as e:
            self.config_parser.print_log_message( 'DEBUG', f"orchestrator: view_worker: Worker {worker_id}: Error creating {view_type_str} {view_detail['source_view_name']}: {e}")
            self.handle_error(e, f"view_worker {worker_id} migrate {view_type_str} {view_detail['source_view_name']}")
            try:
                worker_target_connection.disconnect()
            except:
                pass
            return False, f"ERROR: {e}"

    def stdwf_migrate_views(self):
        self.migrator_tables.insert_main({'task_name': 'Orchestrator', 'subtask_name': 'views migration'})

        if self.config_parser.should_migrate_views():
            self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_views: Migrating views.")

            all_views = self.migrator_tables.fetch_all_views()
            self.config_parser.print_log_message('DEBUG', f"orchestrator: run_migrate_views: found views: {len(all_views)}")
            if all_views:
                normal_views = []
                alias_views = []
                for one_view in all_views:
                    view_detail = self.migrator_tables.decode_view_row(one_view)
                    if view_detail.get('alias_view', False):
                        alias_views.append(view_detail)
                    else:
                        normal_views.append(view_detail)

                max_workers = self.config_parser.get_parallel_workers_count()

                # Process normal views first
                if normal_views:
                    self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_views: Starting parallel creation of normal views.")
                    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
                        futures = {executor.submit(self.view_worker, view_detail): view_detail for view_detail in normal_views}
                        for future in concurrent.futures.as_completed(futures):
                            view_detail = futures[future]
                            try:
                                success, message = future.result()
                                if success:
                                    self.migrator_tables.update_view_status({'row_id': view_detail['id'], 'success': True, 'message': 'migrated OK'})
                                    self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_views: View {view_detail['source_view_name']} migrated successfully.")
                                else:
                                    self.config_parser.print_log_message('ERROR', f"orchestrator: run_migrate_views: Error migrating view {view_detail['source_view_name']}.")
                                    self.migrator_tables.update_view_status({'row_id': view_detail['id'], 'success': False, 'message': message})
                            except Exception as e:
                                self.config_parser.print_log_message('ERROR', f"orchestrator: run_migrate_views: Exception migrating view {view_detail['source_view_name']}: {e}")
                                self.migrator_tables.update_view_status({'row_id': view_detail['id'], 'success': False, 'message': f'ERROR: {e}'})

                # Process alias views after all normal views have completed
                if alias_views:
                    self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_views: Starting parallel creation of alias views.")
                    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
                        futures = {executor.submit(self.view_worker, view_detail): view_detail for view_detail in alias_views}
                        for future in concurrent.futures.as_completed(futures):
                            view_detail = futures[future]
                            try:
                                success, message = future.result()
                                if success:
                                    self.migrator_tables.update_view_status({'row_id': view_detail['id'], 'success': True, 'message': 'migrated OK'})
                                    self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_views: Alias view {view_detail['source_view_name']} migrated successfully.")
                                else:
                                    self.config_parser.print_log_message('ERROR', f"orchestrator: run_migrate_views: Error migrating alias view {view_detail['source_view_name']}.")
                                    self.migrator_tables.update_view_status({'row_id': view_detail['id'], 'success': False, 'message': message})
                            except Exception as e:
                                self.config_parser.print_log_message('ERROR', f"orchestrator: run_migrate_views: Exception migrating alias view {view_detail['source_view_name']}: {e}")
                                self.migrator_tables.update_view_status({'row_id': view_detail['id'], 'success': False, 'message': f'ERROR: {e}'})
            else:
                self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_views: No views found to migrate.")
        else:
            self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_views: Skipping view migration as requested.")
        self.migrator_tables.update_main_status({'task_name': 'Orchestrator', 'subtask_name': 'views migration', 'success': True, 'message': 'finished OK'})

    def stdwf_migrate_comments(self):
        self.migrator_tables.insert_main({'task_name': 'Orchestrator', 'subtask_name': 'comments migration'})
        self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_comments: Migrating comments.")
        all_tables = self.migrator_tables.fetch_all_tables()
        self.target_connection.connect()
        use_aliases_as_target_names = self.config_parser.get_use_aliases_as_target_names()

        try:
            for table_detail in all_tables:
                table_data = self.migrator_tables.decode_table_row(table_detail)
                if table_data['table_comment']:
                    target_table_name = table_data['target_alias_name'] if use_aliases_as_target_names and table_data.get('target_alias_name') else table_data['target_table_name']
                    # Escape single quotes in the comment to prevent SQL injection
                    safe_table_comment = table_data['table_comment'].replace("'", "''")
                    query = f"""COMMENT ON TABLE
                    "{table_data['target_schema_name']}"."{self.config_parser.convert_names_case(target_table_name)}"
                    IS '{safe_table_comment}'"""
                    self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_comments: Setting comment for table {table_data['target_table_name']} in target database.")
                    self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_comments: Executing comment query: {query}")
                    self.target_connection.execute_query(query)

                for col in table_data['target_columns'].keys():
                    column_comment = table_data['target_columns'][col]['column_comment']
                    if column_comment:
                        target_table_name = table_data['target_alias_name'] if use_aliases_as_target_names and table_data.get('target_alias_name') else table_data['target_table_name']
                        target_column_name = table_data['target_columns'][col].get('target_alias_name') if use_aliases_as_target_names and table_data['target_columns'][col].get('target_alias_name') else table_data['target_columns'][col]['column_name']
                        # Escape single quotes in the comment to prevent SQL injection
                        safe_column_comment = table_data['target_columns'][col]['column_comment'].replace("'", "''")
                        query = f"""COMMENT ON COLUMN
                        "{table_data['target_schema_name']}"."{self.config_parser.convert_names_case(target_table_name)}"."{self.config_parser.convert_names_case(target_column_name)}"
                        IS '{safe_column_comment}'"""
                        self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_comments: Setting comment for column {table_data['target_columns'][col]['column_name']} in target database.")
                        self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_comments: Executing comment query: {query}")
                        self.target_connection.execute_query(query)

            all_indexes = self.migrator_tables.fetch_all_indexes()
            for index_detail in all_indexes:
                index_data = self.migrator_tables.decode_index_row(index_detail)
                if index_data['index_comment']:
                    target_table_name = index_data['target_alias_name'] if use_aliases_as_target_names and index_data.get('target_alias_name') else index_data['target_table_name']
                    # Escape single quotes in the comment to prevent SQL injection
                    safe_index_comment = index_data['index_comment'].replace("'", "''")
                    index_name = f"{index_data['index_name']}_tab_{target_table_name}"
                    query = f"""COMMENT ON INDEX
                    "{index_data['target_schema_name']}"."{self.config_parser.convert_names_case(index_name)}"
                    IS '{safe_index_comment}'"""
                    self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_comments: Setting comment for index {index_name} in target database.")
                    self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_comments: Executing comment query: {query}")
                    self.target_connection.execute_query(query)

            all_constraints = self.migrator_tables.fetch_all_constraints()
            for constraint_detail in all_constraints:
                constraint_data = self.migrator_tables.decode_constraint_row(constraint_detail)
                if constraint_data['constraint_comment']:
                    target_table_name = constraint_data['target_alias_name'] if use_aliases_as_target_names and constraint_data.get('target_alias_name') else constraint_data['target_table_name']
                    # Escape single quotes in the comment to prevent SQL injection
                    safe_constraint_comment = constraint_data['constraint_comment'].replace("'", "''")
                    query = f"""COMMENT ON CONSTRAINT
                    "{self.config_parser.convert_names_case(constraint_data['constraint_name'])}"
                    ON "{constraint_data['target_schema_name']}"."{self.config_parser.convert_names_case(target_table_name)}"
                    IS '{safe_constraint_comment}'"""
                    self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_comments: Setting comment for constraint {constraint_data['constraint_name']} in target database.")
                    self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_comments: Executing comment query: {query}")
                    self.target_connection.execute_query(query)

            all_triggers = self.migrator_tables.fetch_all_triggers()
            for trigger_detail in all_triggers:
                trigger_data = self.migrator_tables.decode_trigger_row(trigger_detail)
                if trigger_data['trigger_comment']:
                    # Escape single quotes in the comment to prevent SQL injection
                    safe_trigger_comment = trigger_data['trigger_comment'].replace("'", "''")
                    query = f"""COMMENT ON TRIGGER
                    "{trigger_data['target_schema_name']}"."{self.config_parser.convert_names_case(trigger_data['trigger_name'])}"
                    IS '{safe_trigger_comment}'"""
                    self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_comments: Setting comment for trigger {trigger_data['trigger_name']} in target database.")
                    self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_comments: Executing comment query: {query}")
                    self.target_connection.execute_query(query)

            all_views = self.migrator_tables.fetch_all_views()
            for view_detail in all_views:
                view_data = self.migrator_tables.decode_view_row(view_detail)
                if view_data['view_comment']:
                    # target_view_name = view_data['target_view_alias'] if use_aliases_as_target_names and view_data.get('target_view_alias') else view_data['target_view_name']
                    target_view_name = self.config_parser.convert_names_case(target_view_name)
                    # Escape single quotes in the comment to prevent SQL injection
                    safe_view_comment = view_data['view_comment'].replace("'", "''")
                    query = f"""COMMENT ON VIEW
                    "{view_data['target_schema_name']}"."{target_view_name}"
                    IS '{safe_view_comment}'"""
                    self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_comments: Setting comment for view {target_view_name} in target database.")
                    self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_comments: Executing comment query: {query}")
                    self.target_connection.execute_query(query)

            all_user_defined_types = self.migrator_tables.fetch_all_user_defined_types()
            for type_detail in all_user_defined_types:
                type_data = self.migrator_tables.decode_user_defined_type_row(type_detail)
                if type_data['type_comment']:
                    # Escape single quotes in the comment to prevent SQL injection
                    safe_type_comment = type_data['type_comment'].replace("'", "''")
                    query = f"""COMMENT ON TYPE
                    "{type_data['target_schema_name']}"."{self.config_parser.convert_names_case(type_data['type_name'])}"
                    IS '{safe_type_comment}'"""
                    self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_comments: Setting comment for user defined type {type_data['type_name']} in target database.")
                    self.config_parser.print_log_message( 'DEBUG', f"orchestrator: run_migrate_comments: Executing comment query: {query}")
                    self.target_connection.execute_query(query)

            self.target_connection.disconnect()
            self.migrator_tables.update_main_status({'task_name': 'Orchestrator', 'subtask_name': 'comments migration', 'success': True, 'message': 'finished OK'})
            self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_comments: Comments migrated successfully.")
        except Exception as e:
            self.migrator_tables.update_main_status({'task_name': 'Orchestrator', 'subtask_name': 'comments migration', 'success': False, 'message': f'ERROR: {e}'})
            self.handle_error(e, 'migrate_comments')
            self.target_connection.disconnect()
            return False

    def handle_error(self, e, description=None):
        self.config_parser.print_log_message('ERROR', f"orchestrator: handle_error: An error in {self.__class__.__name__} ({description}): {e}")
        self.config_parser.print_log_message('ERROR', traceback.format_exc())
        if self.on_error_action == 'stop':
            self.config_parser.print_log_message('ERROR', "orchestrator: handle_error: Stopping due to error.")
            exit(1)
        else:
            self.config_parser.print_log_message('WARNING', f"orchestrator: handle_error: Error caught, but continuing as requested by configuration (on_error_action='{self.on_error_action}').")

    def check_pausing_resuming(self):
        if self.config_parser.pause_migration_fired():
            self.config_parser.print_log_message('INFO', f"orchestrator: check_pausing_resuming: paused. Waiting for resume signal...")
            self.config_parser.wait_for_resume()
            self.config_parser.print_log_message('INFO', f"orchestrator: check_pausing_resuming: resumed.")

    def stdwf_migrate_sequences(self):
        self.migrator_tables.insert_main({'task_name': 'Orchestrator', 'subtask_name': 'sequences migration'})
        workers_requested = self.config_parser.get_parallel_workers_count()
        settings = {
            'source_db_type': self.config_parser.get_source_db_type(),
            'target_db_type': self.config_parser.get_target_db_type(),
            'migrator_tables': self.migrator_tables,
        }

        self.config_parser.print_log_message('INFO', f"orchestrator: run_migrate_sequences: Starting {workers_requested} parallel workers to create sequences in target database.")

        migrate_sequences = self.migrator_tables.fetch_all_sequences(self.config_parser.is_resume_after_crash())

        if len(migrate_sequences) > 0:
            with concurrent.futures.ThreadPoolExecutor(max_workers=workers_requested) as executor:
                futures = {}
                for sequence_row in migrate_sequences:
                    sequence_data = self.migrator_tables.decode_sequence_row(sequence_row)
                    self.config_parser.print_log_message('DEBUG3', f"orchestrator: run_migrate_sequences: futures running count: {len(futures)}")
                    while len(futures) >= workers_requested:
                        done, _ = concurrent.futures.wait(futures, return_when=concurrent.futures.FIRST_COMPLETED)
                        for future in done:
                            sequence_done = futures.pop(future)
                            if future.result() == False:
                                if self.on_error_action == 'stop':
                                    self.config_parser.print_log_message('ERROR', "orchestrator: run_migrate_sequences: Stopping execution due to error.")
                                    exit(1)
                            else:
                                self.migrator_tables.update_sequence_status({'sequence_id': sequence_done['sequence_id'], 'success': True, 'message': 'migrated OK'})

                    future = executor.submit(self.sequence_worker, sequence_data, settings)
                    futures[future] = sequence_data

                self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_sequences: Processing remaining futures")
                for future in concurrent.futures.as_completed(futures):
                    sequence_done = futures[future]
                    if future.result() == False:
                        if self.on_error_action == 'stop':
                            self.config_parser.print_log_message('ERROR', "orchestrator: run_migrate_sequences: Stopping execution due to error.")
                            exit(1)
                    else:
                        self.migrator_tables.update_sequence_status({'sequence_id': sequence_done['sequence_id'], 'success': True, 'message': 'migrated OK'})

            self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_sequences: Sequences processed successfully.")
        else:
            self.config_parser.print_log_message('INFO', "orchestrator: run_migrate_sequences: No sequences to create.")

        self.migrator_tables.update_main_status({'task_name': 'Orchestrator', 'subtask_name': 'sequences migration', 'success': True, 'message': 'finished OK'})

    def sequence_worker(self, sequence_data, settings):
        worker_id = uuid.uuid4()
        try:
            target_sequence_name = sequence_data['target_sequence_name']
            self.migrator_tables.update_protocol_task_started('sequences', sequence_data['sequence_id'])
            self.config_parser.print_log_message('INFO', f"orchestrator: sequence_worker: Worker {worker_id}: Creating sequence {target_sequence_name} in target database.")

            worker_target_connection = self.load_connector('target')
            worker_target_connection.connect()

            result = self.source_connection.migrate_sequences(worker_target_connection, sequence_data)

            worker_target_connection.disconnect()

            if not result:
                self.config_parser.print_log_message('ERROR', f"orchestrator: sequence_worker: Worker {worker_id}: Error migrating sequence {target_sequence_name}.")
                self.migrator_tables.update_sequence_status({'sequence_id': sequence_data['sequence_id'], 'success': False, 'message': 'ERROR: migration failed in connector'})
                return False

            self.config_parser.print_log_message('INFO', f"orchestrator: sequence_worker: Worker {worker_id}: Sequence '{target_sequence_name}' created successfully.")
            return True

        except Exception as e:
            self.config_parser.print_log_message('ERROR', f"orchestrator: sequence_worker: Worker {worker_id}: Exception migrating sequence: {e}")
            self.migrator_tables.update_sequence_status({'sequence_id': sequence_data['sequence_id'], 'success': False, 'message': f'ERROR: {e}'})
            return False

if __name__ == "__main__":
    print("This script is not meant to be run directly")
