from __future__ import annotations

from collections import Counter
from collections.abc import Callable, Mapping
from enum import IntEnum
import logging
import os
import re
import shlex
import subprocess
from typing import Any, Collection, Generator, Iterable, Literal

from jinja2 import TemplateError
from prompt_toolkit.completion import CompleteEvent, Completer, Completion
from prompt_toolkit.completion.base import Document
from pygments.lexers._mysql_builtins import MYSQL_DATATYPES, MYSQL_FUNCTIONS, MYSQL_KEYWORDS
import rapidfuzz

from mycli.compat import WIN
from mycli.packages.completion_engine import is_inside_quotes, suggest_type
from mycli.packages.filepaths import complete_path, parse_path, suggest_path
from mycli.packages.polars_completion import complete_polars_transform
from mycli.packages.ptoolkit.history import frecency_score
from mycli.packages.special import llm
from mycli.packages.special.dsn_aliases import DsnAliases
from mycli.packages.special.favoritequeries import (
    FavoriteQueries,
    favorite_query_variable_pattern,
    find_favorite_query_template_keys,
)
from mycli.packages.special.main import COMMANDS as SPECIAL_COMMANDS
from mycli.packages.sql_utils import extract_columns_from_select, extract_tables, last_word

_logger = logging.getLogger(__name__)
_CASE_CHANGE_PAT = re.compile('(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])')
_INDEXED_COLUMN_STYLE = 'class:completion-menu.completion.indexed'


class Fuzziness(IntEnum):
    PERFECT = 0
    REGEX = 1
    UNDER_WORDS = 2
    CAMEL_CASE = 3
    RAPIDFUZZ = 4


class SQLCompleter(Completer):
    favorite_keywords = [
        'SELECT',
        'FROM',
        'WHERE',
        'UPDATE',
        'DELETE FROM',
        'GROUP BY',
        'ORDER BY',
        'JOIN',
        'LEFT JOIN',
        'INSERT INTO',
        'LIKE',
        'LIMIT',
        'WITH',
        'EXPLAIN',
    ]
    keywords_raw = [
        x.upper()
        for x in favorite_keywords
        + list(MYSQL_DATATYPES)
        + list(MYSQL_KEYWORDS)
        + ['ALTER TABLE', 'CHANGE MASTER TO', 'CHARACTER SET', 'FOREIGN KEY']
    ]
    keywords_d = dict.fromkeys(keywords_raw)
    for x in SPECIAL_COMMANDS:
        if x.upper() in keywords_d:
            del keywords_d[x.upper()]
    keywords = list(keywords_d)

    tidb_keywords = [
        "SELECT",
        "FROM",
        "WHERE",
        "DELETE FROM",
        "UPDATE",
        "GROUP BY",
        "JOIN",
        "INSERT INTO",
        "LIKE",
        "LIMIT",
        "ACCOUNT",
        "ACTION",
        "ADD",
        "ADDDATE",
        "ADMIN",
        "ADVISE",
        "AFTER",
        "AGAINST",
        "AGO",
        "ALGORITHM",
        "ALL",
        "ALTER",
        "ALWAYS",
        "ANALYZE",
        "AND",
        "ANY",
        "APPROX_COUNT_DISTINCT",
        "APPROX_PERCENTILE",
        "AS",
        "ASC",
        "ASCII",
        "ATTRIBUTES",
        "AUTO_ID_CACHE",
        "AUTO_INCREMENT",
        "AUTO_RANDOM",
        "AUTO_RANDOM_BASE",
        "AVG",
        "AVG_ROW_LENGTH",
        "BACKEND",
        "BACKUP",
        "BACKUPS",
        "BATCH",
        "BEGIN",
        "BERNOULLI",
        "BETWEEN",
        "BIGINT",
        "BINARY",
        "BINDING",
        "BINDINGS",
        "BINDING_CACHE",
        "BINLOG",
        "BIT",
        "BIT_AND",
        "BIT_OR",
        "BIT_XOR",
        "BLOB",
        "BLOCK",
        "BOOL",
        "BOOLEAN",
        "BOTH",
        "BOUND",
        "BRIEF",
        "BTREE",
        "BUCKETS",
        "BUILTINS",
        "BY",
        "BYTE",
        "CACHE",
        "CALL",
        "CANCEL",
        "CAPTURE",
        "CARDINALITY",
        "CASCADE",
        "CASCADED",
        "CASE",
        "CAST",
        "CAUSAL",
        "CHAIN",
        "CHANGE",
        "CHAR",
        "CHARACTER",
        "CHARSET",
        "CHECK",
        "CHECKPOINT",
        "CHECKSUM",
        "CIPHER",
        "CLEANUP",
        "CLIENT",
        "CLIENT_ERRORS_SUMMARY",
        "CLUSTERED",
        "CMSKETCH",
        "COALESCE",
        "COLLATE",
        "COLLATION",
        "COLUMN",
        "COLUMNS",
        "COLUMN_FORMAT",
        "COLUMN_STATS_USAGE",
        "COMMENT",
        "COMMIT",
        "COMMITTED",
        "COMPACT",
        "COMPRESSED",
        "COMPRESSION",
        "CONCURRENCY",
        "CONFIG",
        "CONNECTION",
        "CONSISTENCY",
        "CONSISTENT",
        "CONSTRAINT",
        "CONSTRAINTS",
        "CONTEXT",
        "CONVERT",
        "COPY",
        "CORRELATION",
        "CPU",
        "CREATE",
        "CROSS",
        "CSV_BACKSLASH_ESCAPE",
        "CSV_DELIMITER",
        "CSV_HEADER",
        "CSV_NOT_NULL",
        "CSV_NULL",
        "CSV_SEPARATOR",
        "CSV_TRIM_LAST_SEPARATORS",
        "CUME_DIST",
        "CURRENT",
        "CURRENT_DATE",
        "CURRENT_ROLE",
        "CURRENT_TIME",
        "CURRENT_TIMESTAMP",
        "CURRENT_USER",
        "CURTIME",
        "CYCLE",
        "DATA",
        "DATABASE",
        "DATABASES",
        "DATE",
        "DATETIME",
        "DATE_ADD",
        "DATE_SUB",
        "DAY",
        "DAY_HOUR",
        "DAY_MICROSECOND",
        "DAY_MINUTE",
        "DAY_SECOND",
        "DDL",
        "DEALLOCATE",
        "DECIMAL",
        "DEFAULT",
        "DEFINER",
        "DELAYED",
        "DELAY_KEY_WRITE",
        "DENSE_RANK",
        "DEPENDENCY",
        "DEPTH",
        "DESC",
        "DESCRIBE",
        "DIRECTORY",
        "DISABLE",
        "DISABLED",
        "DISCARD",
        "DISK",
        "DISTINCT",
        "DISTINCTROW",
        "DIV",
        "DO",
        "DOT",
        "DOUBLE",
        "DRAINER",
        "DROP",
        "DRY",
        "DUAL",
        "DUMP",
        "DUPLICATE",
        "DYNAMIC",
        "ELSE",
        "ENABLE",
        "ENABLED",
        "ENCLOSED",
        "ENCRYPTION",
        "END",
        "ENFORCED",
        "ENGINE",
        "ENGINES",
        "ENUM",
        "ERROR",
        "ERRORS",
        "ESCAPE",
        "ESCAPED",
        "EVENT",
        "EVENTS",
        "EVOLVE",
        "EXACT",
        "EXCEPT",
        "EXCHANGE",
        "EXCLUSIVE",
        "EXECUTE",
        "EXISTS",
        "EXPANSION",
        "EXPIRE",
        "EXPLAIN",
        "EXPR_PUSHDOWN_BLACKLIST",
        "EXTENDED",
        "EXTRACT",
        "FALSE",
        "FAST",
        "FAULTS",
        "FETCH",
        "FIELDS",
        "FILE",
        "FIRST",
        "FIRST_VALUE",
        "FIXED",
        "FLASHBACK",
        "FLOAT",
        "FLUSH",
        "FOLLOWER",
        "FOLLOWERS",
        "FOLLOWER_CONSTRAINTS",
        "FOLLOWING",
        "FOR",
        "FORCE",
        "FOREIGN",
        "FORMAT",
        "FULL",
        "FULLTEXT",
        "FUNCTION",
        "GENERAL",
        "GENERATED",
        "GET_FORMAT",
        "GLOBAL",
        "GRANT",
        "GRANTS",
        "GROUPS",
        "GROUP_CONCAT",
        "HASH",
        "HAVING",
        "HELP",
        "HIGH_PRIORITY",
        "HISTOGRAM",
        "HISTOGRAMS_IN_FLIGHT",
        "HISTORY",
        "HOSTS",
        "HOUR",
        "HOUR_MICROSECOND",
        "HOUR_MINUTE",
        "HOUR_SECOND",
        "IDENTIFIED",
        "IF",
        "IGNORE",
        "IMPORT",
        "IMPORTS",
        "IN",
        "INCREMENT",
        "INCREMENTAL",
        "INDEX",
        "INDEXES",
        "INFILE",
        "INNER",
        "INPLACE",
        "INSERT_METHOD",
        "INSTANCE",
        "INSTANT",
        "INT",
        "INT1",
        "INT2",
        "INT3",
        "INT4",
        "INT8",
        "INTEGER",
        "INTERNAL",
        "INTERSECT",
        "INTERVAL",
        "INTO",
        "INVISIBLE",
        "INVOKER",
        "IO",
        "IPC",
        "IS",
        "ISOLATION",
        "ISSUER",
        "JOB",
        "JOBS",
        "JSON",
        "JSON_ARRAYAGG",
        "JSON_OBJECTAGG",
        "KEY",
        "KEYS",
        "KEY_BLOCK_SIZE",
        "KILL",
        "LABELS",
        "LAG",
        "LANGUAGE",
        "LAST",
        "LASTVAL",
        "LAST_BACKUP",
        "LAST_VALUE",
        "LEAD",
        "LEADER",
        "LEADER_CONSTRAINTS",
        "LEADING",
        "LEARNER",
        "LEARNERS",
        "LEARNER_CONSTRAINTS",
        "LEFT",
        "LESS",
        "LEVEL",
        "LINEAR",
        "LINES",
        "LIST",
        "LOAD",
        "LOCAL",
        "LOCALTIME",
        "LOCALTIMESTAMP",
        "LOCATION",
        "LOCK",
        "LOCKED",
        "LOGS",
        "LONG",
        "LONGBLOB",
        "LONGTEXT",
        "LOW_PRIORITY",
        "MASTER",
        "MATCH",
        "MAX",
        "MAXVALUE",
        "MAX_CONNECTIONS_PER_HOUR",
        "MAX_IDXNUM",
        "MAX_MINUTES",
        "MAX_QUERIES_PER_HOUR",
        "MAX_ROWS",
        "MAX_UPDATES_PER_HOUR",
        "MAX_USER_CONNECTIONS",
        "MB",
        "MEDIUMBLOB",
        "MEDIUMINT",
        "MEDIUMTEXT",
        "MEMORY",
        "MERGE",
        "MICROSECOND",
        "MIN",
        "MINUTE",
        "MINUTE_MICROSECOND",
        "MINUTE_SECOND",
        "MINVALUE",
        "MIN_ROWS",
        "MOD",
        "MODE",
        "MODIFY",
        "MONTH",
        "NAMES",
        "NATIONAL",
        "NATURAL",
        "NCHAR",
        "NEVER",
        "NEXT",
        "NEXTVAL",
        "NEXT_ROW_ID",
        "NO",
        "NOCACHE",
        "NOCYCLE",
        "NODEGROUP",
        "NODE_ID",
        "NODE_STATE",
        "NOMAXVALUE",
        "NOMINVALUE",
        "NONCLUSTERED",
        "NONE",
        "NORMAL",
        "NOT",
        "NOW",
        "NOWAIT",
        "NO_WRITE_TO_BINLOG",
        "NTH_VALUE",
        "NTILE",
        "NULL",
        "NULLS",
        "NUMERIC",
        "NVARCHAR",
        "OF",
        "OFF",
        "OFFSET",
        "ON",
        "ONLINE",
        "ONLY",
        "ON_DUPLICATE",
        "OPEN",
        "OPTIMISTIC",
        "OPTIMIZE",
        "OPTION",
        "OPTIONAL",
        "OPTIONALLY",
        "OPT_RULE_BLACKLIST",
        "OR",
        "ORDER",
        "OUTER",
        "OUTFILE",
        "OVER",
        "PACK_KEYS",
        "PAGE",
        "PARSER",
        "PARTIAL",
        "PARTITION",
        "PARTITIONING",
        "PARTITIONS",
        "PASSWORD",
        "PERCENT",
        "PERCENT_RANK",
        "PER_DB",
        "PER_TABLE",
        "PESSIMISTIC",
        "PLACEMENT",
        "PLAN",
        "PLAN_CACHE",
        "PLUGINS",
        "POLICY",
        "POSITION",
        "PRECEDING",
        "PRECISION",
        "PREDICATE",
        "PREPARE",
        "PRESERVE",
        "PRE_SPLIT_REGIONS",
        "PRIMARY",
        "PRIMARY_REGION",
        "PRIVILEGES",
        "PROCEDURE",
        "PROCESS",
        "PROCESSLIST",
        "PROFILE",
        "PROFILES",
        "PROXY",
        "PUMP",
        "PURGE",
        "QUARTER",
        "QUERIES",
        "QUERY",
        "QUICK",
        "RANGE",
        "RANK",
        "RATE_LIMIT",
        "READ",
        "REAL",
        "REBUILD",
        "RECENT",
        "RECOVER",
        "RECURSIVE",
        "REDUNDANT",
        "REFERENCES",
        "REGEXP",
        "REGION",
        "REGIONS",
        "RELEASE",
        "RELOAD",
        "REMOVE",
        "RENAME",
        "REORGANIZE",
        "REPAIR",
        "REPEAT",
        "REPEATABLE",
        "REPLACE",
        "REPLAYER",
        "REPLICA",
        "REPLICAS",
        "REPLICATION",
        "REQUIRE",
        "REQUIRED",
        "RESET",
        "RESPECT",
        "RESTART",
        "RESTORE",
        "RESTORES",
        "RESTRICT",
        "RESUME",
        "REVERSE",
        "REVOKE",
        "RIGHT",
        "RLIKE",
        "ROLE",
        "ROLLBACK",
        "ROUTINE",
        "ROW",
        "ROWS",
        "ROW_COUNT",
        "ROW_FORMAT",
        "ROW_NUMBER",
        "RTREE",
        "RUN",
        "RUNNING",
        "S3",
        "SAMPLERATE",
        "SAMPLES",
        "SAN",
        "SAVEPOINT",
        "SCHEDULE",
        "SECOND",
        "SECONDARY_ENGINE",
        "SECONDARY_LOAD",
        "SECONDARY_UNLOAD",
        "SECOND_MICROSECOND",
        "SECURITY",
        "SEND_CREDENTIALS_TO_TIKV",
        "SEPARATOR",
        "SEQUENCE",
        "SERIAL",
        "SERIALIZABLE",
        "SESSION",
        "SESSION_STATES",
        "SET",
        "SETVAL",
        "SHARD_ROW_ID_BITS",
        "SHARE",
        "SHARED",
        "SHOW",
        "SHUTDOWN",
        "SIGNED",
        "SIMPLE",
        "SKIP",
        "SKIP_SCHEMA_FILES",
        "SLAVE",
        "SLOW",
        "SMALLINT",
        "SNAPSHOT",
        "SOME",
        "SOURCE",
        "SPATIAL",
        "SPLIT",
        "SQL",
        "SQL_BIG_RESULT",
        "SQL_BUFFER_RESULT",
        "SQL_CACHE",
        "SQL_CALC_FOUND_ROWS",
        "SQL_NO_CACHE",
        "SQL_SMALL_RESULT",
        "SQL_TSI_DAY",
        "SQL_TSI_HOUR",
        "SQL_TSI_MINUTE",
        "SQL_TSI_MONTH",
        "SQL_TSI_QUARTER",
        "SQL_TSI_SECOND",
        "SQL_TSI_WEEK",
        "SQL_TSI_YEAR",
        "SSL",
        "STALENESS",
        "START",
        "STARTING",
        "STATISTICS",
        "STATS",
        "STATS_AUTO_RECALC",
        "STATS_BUCKETS",
        "STATS_COL_CHOICE",
        "STATS_COL_LIST",
        "STATS_EXTENDED",
        "STATS_HEALTHY",
        "STATS_HISTOGRAMS",
        "STATS_META",
        "STATS_OPTIONS",
        "STATS_PERSISTENT",
        "STATS_SAMPLE_PAGES",
        "STATS_SAMPLE_RATE",
        "STATS_TOPN",
        "STATUS",
        "STD",
        "STDDEV",
        "STDDEV_POP",
        "STDDEV_SAMP",
        "STOP",
        "STORAGE",
        "STORED",
        "STRAIGHT_JOIN",
        "STRICT",
        "STRICT_FORMAT",
        "STRONG",
        "SUBDATE",
        "SUBJECT",
        "SUBPARTITION",
        "SUBPARTITIONS",
        "SUBSTRING",
        "SUM",
        "SUPER",
        "SWAPS",
        "SWITCHES",
        "SYSTEM",
        "SYSTEM_TIME",
        "TABLE",
        "TABLES",
        "TABLESAMPLE",
        "TABLESPACE",
        "TABLE_CHECKSUM",
        "TARGET",
        "TELEMETRY",
        "TELEMETRY_ID",
        "TEMPORARY",
        "TEMPTABLE",
        "TERMINATED",
        "TEXT",
        "THAN",
        "THEN",
        "TIDB",
        "TIFLASH",
        "TIKV_IMPORTER",
        "TIME",
        "TIMESTAMP",
        "TIMESTAMPADD",
        "TIMESTAMPDIFF",
        "TINYBLOB",
        "TINYINT",
        "TINYTEXT",
        "TLS",
        "TO",
        "TOKUDB_DEFAULT",
        "TOKUDB_FAST",
        "TOKUDB_LZMA",
        "TOKUDB_QUICKLZ",
        "TOKUDB_SMALL",
        "TOKUDB_SNAPPY",
        "TOKUDB_UNCOMPRESSED",
        "TOKUDB_ZLIB",
        "TOP",
        "TOPN",
        "TRACE",
        "TRADITIONAL",
        "TRAILING",
        "TRANSACTION",
        "TRIGGER",
        "TRIGGERS",
        "TRIM",
        "TRUE",
        "TRUE_CARD_COST",
        "TRUNCATE",
        "TYPE",
        "UNBOUNDED",
        "UNCOMMITTED",
        "UNDEFINED",
        "UNICODE",
        "UNION",
        "UNIQUE",
        "UNKNOWN",
        "UNLOCK",
        "UNSIGNED",
        "USAGE",
        "USE",
        "USER",
        "USING",
        "UTC_DATE",
        "UTC_TIME",
        "UTC_TIMESTAMP",
        "VALIDATION",
        "VALUE",
        "VALUES",
        "VARBINARY",
        "VARCHAR",
        "VARCHARACTER",
        "VARIABLES",
        "VARIANCE",
        "VARYING",
        "VAR_POP",
        "VAR_SAMP",
        "VERBOSE",
        "VIEW",
        "VIRTUAL",
        "VISIBLE",
        "VOTER",
        "VOTERS",
        "VOTER_CONSTRAINTS",
        "WAIT",
        "WARNINGS",
        "WEEK",
        "WEIGHT_STRING",
        "WHEN",
        "WIDTH",
        "WINDOW",
        "WITH",
        "WITHOUT",
        "WRITE",
        "X509",
        "XOR",
        "YEAR",
        "YEAR_MONTH",
        "ZEROFILL",
    ]

    # misclassified as keywords
    # do they need to also be subtracted from keywords?
    pygments_misclassified_functions = [
        'ASCII',
        'AVG',
        'CHARSET',
        'COALESCE',
        'COLLATION',
        'CONVERT',
        'CUME_DIST',
        'CURRENT_DATE',
        'CURRENT_TIME',
        'CURRENT_TIMESTAMP',
        'CURRENT_USER',
        'DATABASE',
        'DAY',
        'DEFAULT',
        'DENSE_RANK',
        'EXISTS',
        'FIRST_VALUE',
        'FORMAT',
        'GEOMCOLLECTION',
        'GET_FORMAT',
        'GROUPING',
        'HOUR',
        'IF',
        'INSERT',
        'INTERVAL',
        'JSON_TABLE',
        'JSON_VALUE',
        'LAG',
        'LAST_VALUE',
        'LEAD',
        'LEFT',
        'LOCALTIME',
        'LOCALTIMESTAMP',
        'MATCH',
        'MICROSECOND',
        'MINUTE',
        'MOD',
        'MONTH',
        'NTH_VALUE',
        'NTILE',
        'PERCENT_RANK',
        'QUARTER',
        'RANK',
        'REPEAT',
        'REPLACE',
        'REVERSE',
        'RIGHT',
        'ROW_COUNT',
        'ROW_NUMBER',
        'SCHEMA',
        'SECOND',
        'TIMESTAMPADD',
        'TIMESTAMPDIFF',
        'TRUNCATE',
        'USER',
        'UTC_DATE',
        'UTC_TIME',
        'UTC_TIMESTAMP',
        'VALUES',
        'WEEK',
        'WEIGHT_STRING',
    ]

    # should case be respected for functions styled as CamelCase?
    pygments_missing_functions = [
        'BINARY',  # deprecated function, but available everywhere
        'CHAR',
        'DATE',
        'DISTANCE',
        'ETAG',
        'GeometryCollection',
        'JSON_DUALITY_OBJECT',
        'LineString',
        'MultiLineString',
        'MultiPoint',
        'MultiPolygon',
        'Point',
        'Polygon',
        'STRING_TO_VECTOR',
        'TIME',
        'TIMESTAMP',
        'VECTOR_DIM',
        'VECTOR_TO_STRING',
        'YEAR',
    ]

    # so far an incomplete list
    # these should be spun out and completed independently from functions in the value position
    pygments_value_position_nonfunction_keywords = [
        'BETWEEN',
        'CASE',
        'DISTINCT',
        'FALSE',
        'NOT',
        'NULL',
        'TRUE',
    ]

    # should https://dev.mysql.com/doc/refman/9.6/en/loadable-function-reference.html also be added?
    pygments_functions_supplemented = sorted(
        [x.upper() for x in MYSQL_FUNCTIONS]
        + [x.upper() for x in pygments_misclassified_functions]
        + [x.upper() for x in pygments_missing_functions]
        + [x.upper() for x in pygments_value_position_nonfunction_keywords]
    )

    favorite_functions = [
        'COUNT',
        'CONVERT',
        'BINARY',
        'CAST',
        'COALESCE',
        'MAX',
        'MIN',
        'SUM',
        'AVG',
        'JSON_EXTRACT',
        'JSON_VALUE',
        'JSON_REMOVE',
        'JSON_SET',
        'CONCAT',
        'GROUP_CONCAT',
        'CHAR_LENGTH',
        'ROUND',
        'FLOOR',
        'CEIL',
        'IF',
        'IFNULL',
        'SUBSTR',
        'SUBSTRING_INDEX',
        'REPLACE',
        'RIGHT',
        'LEFT',
        'UNIX_TIMESTAMP',
        'FROM_UNIXTIME',
        'RAND',
        'DATEDIFF',
        'DATE_SUB',
    ]
    functions_raw = favorite_functions + pygments_functions_supplemented
    functions = list(dict.fromkeys(functions_raw))

    # https://docs.pingcap.com/tidb/dev/tidb-functions
    tidb_functions = [
        "TIDB_BOUNDED_STALENESS",
        "TIDB_DECODE_KEY",
        "TIDB_DECODE_PLAN",
        "TIDB_IS_DDL_OWNER",
        "TIDB_PARSE_TSO",
        "TIDB_VERSION",
        "TIDB_DECODE_SQL_DIGESTS",
        "VITESS_HASH",
        "TIDB_SHARD",
    ]

    show_items: list[Completion] = []

    change_items = [
        "MASTER_BIND",
        "MASTER_HOST",
        "MASTER_USER",
        "MASTER_PASSWORD",
        "MASTER_PORT",
        "MASTER_CONNECT_RETRY",
        "MASTER_HEARTBEAT_PERIOD",
        "MASTER_LOG_FILE",
        "MASTER_LOG_POS",
        "RELAY_LOG_FILE",
        "RELAY_LOG_POS",
        "MASTER_SSL",
        "MASTER_SSL_CA",
        "MASTER_SSL_CAPATH",
        "MASTER_SSL_CERT",
        "MASTER_SSL_KEY",
        "MASTER_SSL_CIPHER",
        "MASTER_SSL_VERIFY_SERVER_CERT",
        "IGNORE_SERVER_IDS",
    ]

    users: list[str] = []

    character_sets: list[str] = []

    collations: list[str] = []

    def __init__(
        self,
        smart_completion: bool = True,
        supported_formats: tuple = (),
        keyword_casing: str = "auto",
        indexed_column_suffix: str = '*',
        config_property_names: Collection[str] = (),
        frecency_provider: Callable[[], Mapping[str, float]] | None = None,
    ) -> None:
        super(self.__class__, self).__init__()
        self.smart_completion = smart_completion
        self.indexed_column_suffix = indexed_column_suffix
        self.config_property_names = tuple(sorted(config_property_names))
        self.frecency_provider = frecency_provider
        self.reserved_words = set()
        for x in self.keywords:
            self.reserved_words.update(x.split())
        self.name_pattern = re.compile(r"^[_a-zA-Z][_a-zA-Z0-9\$]*$")

        self.special_commands: list[str] = []
        self.special_command_snippets: dict[str, str] = {}
        self.table_formats = supported_formats
        if keyword_casing not in ("upper", "lower", "auto"):
            keyword_casing = "auto"
        self.keyword_casing = keyword_casing
        self.reset_completions()

    def escape_name(self, name: str) -> str:
        if name and ((not self.name_pattern.match(name)) or (name.upper() in self.reserved_words) or (name.upper() in self.functions)):
            name = f'`{name}`'

        return name

    def escaped_names(self, names: Collection[str]) -> list[str]:
        return [self.escape_name(name) for name in names]

    def extend_special_commands(self, special_commands: Mapping[str, str]) -> None:
        # Special commands are not part of all_completions since they can only
        # be at the beginning of a line.
        self.special_commands.extend(special_commands)
        self.special_command_snippets.update(special_commands)

    def extend_database_names(self, databases: list[str]) -> None:
        self.databases.extend([self.escape_name(db) for db in databases])

    def extend_keywords(self, keywords: list[str], replace: bool = False) -> None:
        if replace:
            self.keywords = keywords
        else:
            self.keywords.extend(keywords)
        self.all_completions.update(keywords)

    def extend_show_items(self, show_items: Iterable[tuple]) -> None:
        for show_item in show_items:
            self.show_items.extend(show_item)
            self.all_completions.update(show_item)

    def extend_change_items(self, change_items: Iterable[tuple]) -> None:
        for change_item in change_items:
            self.change_items.extend(change_item)
            self.all_completions.update(change_item)

    def extend_users(self, users: Iterable[tuple]) -> None:
        for user in users:
            self.users.extend(user)
            self.all_completions.update(user)

    def extend_schemata(self, schema: str | None) -> None:
        if schema is None:
            return
        metadata = self.dbmetadata["tables"]
        metadata[schema] = {}

        # dbmetadata.values() are the 'tables' and 'functions' dicts
        for metadata in self.dbmetadata.values():
            metadata[schema] = {}
        self.all_completions.update(schema)

    def extend_relations(self, data: list[tuple[str, str]], kind: Literal['tables', 'views']) -> None:
        """Extend metadata for tables or views

        :param data: list of (rel_name, ) tuples
        :param kind: either 'tables' or 'views'
        :return:
        """
        data_ll = [self.escaped_names(d) for d in data]

        # dbmetadata['tables'][$schema_name][$table_name] should be a list of
        # column names. Default to an asterisk
        metadata = self.dbmetadata[kind]
        for relname in data_ll:
            try:
                metadata[self.dbname][relname[0]] = ["*"]
            except KeyError:
                _logger.error("%r %r listed in unrecognized schema %r", kind, relname[0], self.dbname)
            self.all_completions.add(relname[0])

    def extend_columns(self, column_data: list[tuple[str, str]], kind: Literal['tables', 'views']) -> None:
        """Extend column metadata

        :param column_data: list of (rel_name, column_name) tuples
        :param kind: either 'tables' or 'views'
        :return:
        """
        column_data_ll = [self.escaped_names(d) for d in column_data]

        metadata = self.dbmetadata[kind]
        for relname, column in column_data_ll:
            if relname not in metadata[self.dbname]:
                _logger.error("relname '%s' was not found in db '%s'", relname, self.dbname)
                # this could happen back when the completer populated via two calls:
                # SHOW TABLES then SELECT table_name, column_name from information_schema.columns
                # it's a slight race, but much more likely on Vitess picking random shards for each.
                # see discussion in https://github.com/dbcli/mycli/pull/1182 (tl;dr - let's keep it)
                continue
            metadata[self.dbname][relname].append(column)
            self.all_completions.add(column)

    def extend_indexed_columns(self, index_data: Iterable[tuple[str, str]]) -> None:
        """Extend metadata for columns that lead an index."""
        metadata = self.dbmetadata["indexed_columns"]
        schema_meta = metadata.setdefault(self.dbname, {})
        for table, column in index_data:
            table = self.escape_name(table)
            column = self.escape_name(column)
            schema_meta.setdefault(table, set()).add(column)

    def extend_enum_values(self, enum_data: Iterable[tuple[str, str, list[str]]]) -> None:
        metadata = self.dbmetadata["enum_values"]
        if self.dbname not in metadata:
            metadata[self.dbname] = {}

        for relname, column, values in enum_data:
            relname_escaped = self.escape_name(relname)
            column_escaped = self.escape_name(column)
            table_meta = metadata[self.dbname].setdefault(relname_escaped, {})
            table_meta[column_escaped] = values

    def extend_foreign_keys(self, fk_data: Iterable[tuple[str, str, str, str]]) -> None:
        """Extend FK metadata.

        :param fk_data: iterable of (table_name, column_name, referenced_table_name, referenced_column_name)
        """
        metadata = self.dbmetadata["foreign_keys"]
        schema_meta = metadata.setdefault(self.dbname, {})
        schema_meta.setdefault("tables", {})
        schema_meta.setdefault("relations", [])
        for table, col, ref_table, ref_col in fk_data:
            table = self.escape_name(table)
            col = self.escape_name(col)
            ref_table = self.escape_name(ref_table)
            ref_col = self.escape_name(ref_col)
            schema_meta["tables"].setdefault(table, set()).add(ref_table)
            schema_meta["tables"].setdefault(ref_table, set()).add(table)
            schema_meta["relations"].append((table, col, ref_table, ref_col))

    def _fk_join_conditions(self, tables: list[tuple[str | None, str, str]]) -> list[str]:
        """Return FK-based join condition strings for the tables currently in the query.

        For each FK relation where both the FK table and the referenced table appear in
        *tables*, yields a string like ``alias1.col = alias2.ref_col`` (using the alias
        when one exists, otherwise the table name).
        """
        schema_meta = self.dbmetadata["foreign_keys"].get(self.dbname, {})
        relations = schema_meta.get("relations", [])

        # Map escaped table name -> alias (or table name when no alias).
        # Skip tables from a different schema; we only have FK metadata for the current db.
        alias_map: dict[str, str] = {}
        for tbl_schema, tbl, alias in tables:
            if tbl_schema and tbl_schema != self.dbname:
                continue
            escaped = self.escape_name(tbl)
            alias_map[escaped] = alias or tbl

        conditions: list[str] = []
        for fk_table, fk_col, ref_table, ref_col in relations:
            lhs = alias_map.get(fk_table)
            rhs = alias_map.get(ref_table)
            if lhs and rhs:
                conditions.append(f"{lhs}.{fk_col} = {rhs}.{ref_col}")
        return conditions

    def extend_functions(self, func_data: list[str] | Generator[tuple[str, str]], builtin: bool = False) -> None:
        # if 'builtin' is set this is extending the list of builtin functions
        if builtin:
            if isinstance(func_data, list):
                self.functions.extend(func_data)
            return

        # 'func_data' is a generator object. It can throw an exception while
        # being consumed. This could happen if the user has launched the app
        # without specifying a database name. This exception must be handled to
        # prevent crashing.
        try:
            func_data_ll = [self.escaped_names(d) for d in func_data]
        except Exception:
            func_data_ll = []

        # dbmetadata['functions'][$schema_name][$function_name] should return
        # function metadata.
        metadata = self.dbmetadata["functions"]

        for func in func_data_ll:
            metadata[self.dbname][func[0]] = None
            self.all_completions.add(func[0])

    def extend_procedures(self, procedure_data: Generator[tuple]) -> None:
        metadata = self.dbmetadata["procedures"]
        if self.dbname not in metadata:
            metadata[self.dbname] = {}

        for elt in procedure_data:
            # not sure why this happens on MariaDB in some cases
            # see https://github.com/dbcli/mycli/issues/1531
            if not elt:
                continue
            if not elt[0]:
                continue
            metadata[self.dbname][elt[0]] = None

    def extend_character_sets(self, character_set_data: Generator[tuple]) -> None:
        for elt in character_set_data:
            if not elt:
                continue
            if not elt[0]:
                continue
            self.character_sets.append(elt[0])
            self.all_completions.update(elt[0])

    def extend_collations(self, collation_data: Generator[tuple]) -> None:
        for elt in collation_data:
            if not elt:
                continue
            if not elt[0]:
                continue
            self.collations.append(elt[0])
            self.all_completions.update(elt[0])

    def set_dbname(self, dbname: str | None) -> None:
        self.dbname = dbname or ''

    def load_schema_metadata(
        self,
        schema: str,
        table_columns: dict[str, list[str]],
        indexed_columns: dict[str, set[str]],
        foreign_keys: dict[str, Any],
        enum_values: dict[str, dict[str, list[str]]],
        functions: dict[str, None],
        procedures: dict[str, None],
    ) -> None:
        """Atomically replace the completion metadata for *schema*.

        Each argument is pre-built by the caller in the same shape that
        ``dbmetadata[kind][schema]`` uses internally.  Replacing the
        per-schema dicts by assignment (rather than appending to the live
        structures) keeps concurrent readers of ``get_completions`` safe.
        """
        if not schema:
            return
        self.dbmetadata["tables"][schema] = table_columns
        self.dbmetadata["indexed_columns"][schema] = indexed_columns
        self.dbmetadata["views"].setdefault(schema, {})
        self.dbmetadata["functions"][schema] = functions
        self.dbmetadata["procedures"][schema] = procedures
        self.dbmetadata["enum_values"][schema] = enum_values
        self.dbmetadata["foreign_keys"][schema] = foreign_keys
        self._register_schema_completions(schema, table_columns, functions)

    def copy_other_schemas_from(self, source: "SQLCompleter", exclude: str | None) -> None:
        """Copy per-schema metadata from *source*, skipping *exclude*.

        After a completion refresh swaps in a fresh completer that was
        populated only with the current schema's data, this restores any
        previously-loaded metadata for other schemas so the user can keep
        using qualified completions (``OtherSchema.table``) without a
        re-fetch.
        """
        kinds = ("tables", "views", "functions", "procedures", "enum_values", "foreign_keys", "indexed_columns")
        for kind in kinds:
            src_map = source.dbmetadata.get(kind, {})
            dest_map = self.dbmetadata.setdefault(kind, {})
            for schema_name, data in src_map.items():
                if not schema_name or schema_name == exclude:
                    continue
                if schema_name in dest_map:
                    continue
                dest_map[schema_name] = data
        for schema_name, table_columns in self.dbmetadata["tables"].items():
            if schema_name == exclude:
                continue
            functions = self.dbmetadata.get("functions", {}).get(schema_name, {})
            self._register_schema_completions(schema_name, table_columns, functions)

    def _register_schema_completions(
        self,
        schema: str,
        table_columns: dict[str, list[str]],
        functions: dict[str, None] | dict[str, Any],
    ) -> None:
        self.all_completions.add(schema)
        for table, cols in table_columns.items():
            self.all_completions.add(table)
            for col in cols:
                if col != "*":
                    self.all_completions.add(col)
        for func_name in functions:
            self.all_completions.add(func_name)

    def reset_completions(self) -> None:
        self.databases: list[str] = []
        self.users: list[str] = []
        self.character_sets: list[str] = []
        self.collations: list[str] = []
        self.show_items: list[Completion] = []
        self.dbname = ""
        self.dbmetadata: dict[str, Any] = {
            "tables": {},
            "views": {},
            "functions": {},
            "procedures": {},
            "enum_values": {},
            "foreign_keys": {},
            "indexed_columns": {},
        }
        self.all_completions = set(self.keywords + self.functions)

    def maybe_quote_identifier(self, item: str) -> str:
        if item.startswith('`'):
            return item
        if item == '*':
            return item
        return '`' + item + '`'

    def quote_collection_if_needed(
        self,
        text: str,
        collection: Collection[Any],
        text_before_cursor: str,
    ) -> Collection[Any]:
        # checking text.startswith() first is an optimization; is_inside_quotes() covers more cases
        if text.startswith('`') or is_inside_quotes(text_before_cursor, len(text_before_cursor)) == 'backtick':
            return [self.maybe_quote_identifier(x) if isinstance(x, str) else x for x in collection]
        return collection

    def word_parts_match(
        self,
        text_parts: list[str],
        item_parts: list[str],
    ) -> bool:
        occurrences = 0
        for text_part in text_parts:
            for item_part in item_parts:
                if item_part.startswith(text_part):
                    occurrences += 1
                    break
        return occurrences >= len(text_parts)

    def find_fuzzy_match(
        self,
        item: str,
        pattern: re.Pattern[str],
        under_words_text: list[str],
        case_words_text: list[str],
    ) -> int | None:
        if pattern.search(item.lower()):
            return Fuzziness.REGEX

        under_words_item = [x for x in item.lower().split('_') if x]
        if self.word_parts_match(under_words_text, under_words_item):
            return Fuzziness.UNDER_WORDS

        case_words_item = re.split(_CASE_CHANGE_PAT, item)
        if self.word_parts_match(case_words_text, case_words_item):
            return Fuzziness.CAMEL_CASE

        return None

    def find_fuzzy_matches(
        self,
        last: str,
        text: str,
        collection: Collection[Any],
    ) -> list[tuple[str, int]]:
        completions: list[tuple[str, int]] = []
        regex = '.{0,3}?'.join(map(re.escape, text))
        pattern = re.compile(f'({regex})')
        under_words_text = [x for x in text.split('_') if x]
        case_words_text = re.split(_CASE_CHANGE_PAT, last)

        for item in collection:
            fuzziness = self.find_fuzzy_match(item, pattern, under_words_text, case_words_text)
            if fuzziness is not None:
                completions.append((item, fuzziness))

        if len(text) >= 4:
            rapidfuzz_matches = rapidfuzz.process.extract(
                text,
                collection,
                scorer=rapidfuzz.fuzz.WRatio,
                # todo: maybe make our own processor which only does case-folding
                # because underscores are valuable info
                processor=rapidfuzz.utils.default_process,
                limit=20,
                score_cutoff=75,
            )
            existing = {c[0] for c in completions}
            for item, _score, _type in rapidfuzz_matches:
                if len(item) < len(text) / 1.5 or item in existing:
                    continue
                completions.append((item, Fuzziness.RAPIDFUZZ))

        return completions

    def find_perfect_matches(
        self,
        text: str,
        collection: Collection[Any],
        start_only: bool,
    ) -> list[tuple[str, int]]:
        completions: list[tuple[str, int]] = []
        match_end_limit = len(text) if start_only else None
        for item in collection:
            match_point = item.lower().find(text, 0, match_end_limit)
            if match_point >= 0:
                completions.append((item, Fuzziness.PERFECT))
        return completions

    def resolve_casing(
        self,
        casing: str | None,
        last: str,
    ) -> str | None:
        if casing != 'auto':
            return casing
        return 'lower' if last and (last[0].islower() or last[-1].islower()) else 'upper'

    def apply_casing(
        self,
        completions: list[tuple[str, int]],
        casing: str | None,
    ) -> Generator[tuple[str, int], None, None]:
        if casing is None:
            return (completion for completion in completions)

        def apply_case(tup: tuple[str, int]) -> tuple[str, int]:
            kw, fuzziness = tup
            if casing == 'upper':
                return (kw.upper(), fuzziness)
            return (kw.lower(), fuzziness)

        return (apply_case(completion) for completion in completions)

    def find_matches(
        self,
        orig_text: str,
        collection: Collection,
        start_only: bool = False,
        fuzzy: bool = True,
        casing: str | None = None,
        text_before_cursor: str = '',
    ) -> Generator[tuple[str, int], None, None]:
        """Find completion matches for the given text.

        Given the user's input text and a collection of available
        completions, find completions matching the last word of the
        text.

        If `start_only` is True, the text will match an available
        completion only at the beginning. Otherwise, a completion is
        considered a match if the text appears anywhere within it.

        yields prompt_toolkit Completion instances for any matches found
        in the collection of available completions.
        """
        last = last_word(orig_text, include='most_punctuations')
        text = last.lower()
        quoted_collection = self.quote_collection_if_needed(text, collection, text_before_cursor)

        if fuzzy:
            completions = self.find_fuzzy_matches(last, text, quoted_collection)
        else:
            completions = self.find_perfect_matches(text, quoted_collection, start_only)

        casing = self.resolve_casing(casing, last)
        return self.apply_casing(completions, casing)

    def get_completions(
        self,
        document: Document,
        complete_event: CompleteEvent | None,
        smart_completion: bool | None = None,
    ) -> Iterable[Completion]:
        polars_completions = complete_polars_transform(document.text_before_cursor)
        if polars_completions is not None:
            return (
                Completion(
                    candidate.text,
                    candidate.start_position,
                    display=candidate.display,
                    display_meta=candidate.display_meta,
                )
                for candidate in polars_completions
            )

        word_before_cursor = document.get_word_before_cursor(WORD=True)
        last_for_len = last_word(word_before_cursor, include="most_punctuations")
        text_for_len = last_for_len.lower()
        last_for_len_paths = last_word(word_before_cursor, include='alphanum_underscore')
        frecency = self.frecency_provider() if self.frecency_provider is not None else {}

        if smart_completion is None:
            smart_completion = self.smart_completion

        # If smart_completion is off then match any word that starts with
        # 'word_before_cursor'.
        if not smart_completion:
            matches: Iterable[tuple[str, int]] = self.find_matches(
                word_before_cursor,
                self.all_completions,
                start_only=True,
                fuzzy=False,
                text_before_cursor=document.text_before_cursor,
            )
            if frecency:
                matches = sorted(matches, key=lambda item: -frecency_score(item[0], frecency))
            return (Completion(x[0], -len(text_for_len)) for x in matches)

        completions: list[tuple[str, int, int]] = []
        indexed_column_candidates: set[str] = set()
        special_command_candidates: set[str] = set()
        suggestions = suggest_type(document.text, document.text_before_cursor)
        rigid_sort = False
        length_based_on_path = False
        source_file_completion_length: int | None = None
        config_property_length: int | None = None
        completion_filter_text = text_for_len

        rank = 0
        for suggestion in suggestions:
            _logger.debug("Suggestion type: %r", suggestion["type"])
            rank += 1

            if suggestion["type"] == "column":
                tables = suggestion["tables"]
                _logger.debug("Completion column scope: %r", tables)
                scoped_cols = self.populate_scoped_cols(tables)
                if suggestion.get("drop_unique"):
                    # drop_unique is used for 'tb11 JOIN tbl2 USING (...'
                    # which should suggest only columns that appear in more than
                    # one table
                    scoped_cols = [col for (col, count) in Counter(scoped_cols).items() if count > 1 and col != "*"]
                elif not tables:
                    # if tables was empty, this is a naked SELECT and we are
                    # showing all columns. So make them unique and sort them.
                    scoped_cols = sorted(set(scoped_cols), key=lambda s: s.strip('`'))

                cols = list(
                    self.find_matches(
                        word_before_cursor,
                        scoped_cols,
                        text_before_cursor=document.text_before_cursor,
                    )
                )
                indexed_columns = {self._strip_backticks(column).casefold() for column in self.populate_scoped_indexed_columns(tables)}
                indexed_column_candidates.update(
                    candidate for candidate, _fuzziness in cols if self._strip_backticks(candidate).casefold() in indexed_columns
                )
                completions.extend([(*x, rank) for x in cols])

            elif suggestion["type"] == "function":
                # suggest user-defined functions using substring matching
                funcs = self.populate_schema_objects(suggestion["schema"], "functions")
                user_funcs = self.find_matches(
                    word_before_cursor,
                    funcs,
                    text_before_cursor=document.text_before_cursor,
                )
                completions.extend([(*x, rank) for x in user_funcs])

                # suggest hardcoded functions using startswith matching only if
                # there is no schema qualifier. If a schema qualifier is
                # present it probably denotes a table.
                # eg: SELECT * FROM users u WHERE u.
                if not suggestion["schema"]:
                    predefined_funcs = self.find_matches(
                        word_before_cursor,
                        self.functions,
                        start_only=True,
                        fuzzy=False,
                        casing=self.keyword_casing,
                        text_before_cursor=document.text_before_cursor,
                    )
                    completions.extend([(*x, rank) for x in predefined_funcs])

            elif suggestion["type"] == "procedure":
                procs = self.populate_schema_objects(suggestion["schema"], "procedures")
                procs_m = self.find_matches(
                    word_before_cursor,
                    procs,
                    text_before_cursor=document.text_before_cursor,
                )
                completions.extend([(*x, rank) for x in procs_m])

            elif suggestion['type'] == 'introducer':
                introducers = [f'_{x}' for x in self.character_sets]
                introducers_m = self.find_matches(
                    word_before_cursor,
                    introducers,
                    text_before_cursor=document.text_before_cursor,
                )
                completions.extend([(*x, rank) for x in introducers_m])

            elif suggestion['type'] == 'character_set':
                charsets_m = self.find_matches(
                    word_before_cursor,
                    self.character_sets,
                    text_before_cursor=document.text_before_cursor,
                )
                completions.extend([(*x, rank) for x in charsets_m])

            elif suggestion['type'] == 'collation':
                collations_m = self.find_matches(
                    word_before_cursor,
                    self.collations,
                    text_before_cursor=document.text_before_cursor,
                )
                completions.extend([(*x, rank) for x in collations_m])

            elif suggestion["type"] == "table":
                # If this is a select and columns are given, parse the columns and
                # then only return tables that have one or more of the given columns.
                # If no columns are given (or able to be parsed), return all tables
                # as usual.
                columns = extract_columns_from_select(document.text)
                if columns:
                    tables = self.populate_schema_objects(suggestion["schema"], "tables", columns)
                else:
                    tables = self.populate_schema_objects(suggestion["schema"], "tables")

                if suggestion.get("join"):
                    # For JOINs, suggest FK-related tables first (lower rank = higher priority)
                    current_tables = extract_tables(document.text)
                    fk_map = self.dbmetadata["foreign_keys"].get(self.dbname, {}).get("tables", {})
                    fk_related: set[str] = set()
                    for tbl_schema, tbl, _alias in current_tables:
                        # Skip cross-schema tables; FK metadata is only for the current db
                        if tbl_schema and tbl_schema != self.dbname:
                            continue
                        escaped = self.escape_name(tbl)
                        fk_related.update(fk_map.get(escaped, set()))
                    fk_tables = [t for t in tables if t in fk_related]
                    other_tables = [t for t in tables if t not in fk_related]
                    fk_tables_m = self.find_matches(
                        word_before_cursor,
                        fk_tables,
                        text_before_cursor=document.text_before_cursor,
                    )
                    other_tables_m = self.find_matches(
                        word_before_cursor,
                        other_tables,
                        text_before_cursor=document.text_before_cursor,
                    )
                    completions.extend([(*x, rank) for x in fk_tables_m])
                    completions.extend([(*x, rank + 1) for x in other_tables_m])
                else:
                    tables_m = self.find_matches(
                        word_before_cursor,
                        tables,
                        text_before_cursor=document.text_before_cursor,
                    )
                    completions.extend([(*x, rank) for x in tables_m])

            elif suggestion["type"] == "view":
                views = self.populate_schema_objects(suggestion["schema"], "views")
                views_m = self.find_matches(
                    word_before_cursor,
                    views,
                    text_before_cursor=document.text_before_cursor,
                )
                completions.extend([(*x, rank) for x in views_m])

            elif suggestion["type"] == "fk_join":
                fk_conditions = self._fk_join_conditions(suggestion["tables"])
                fk_conditions_m = self.find_matches(
                    word_before_cursor,
                    fk_conditions,
                    text_before_cursor=document.text_before_cursor,
                )
                completions.extend([(*x, rank) for x in fk_conditions_m])

            elif suggestion["type"] == "alias":
                aliases = suggestion["aliases"]
                aliases_m = self.find_matches(
                    word_before_cursor,
                    aliases,
                    text_before_cursor=document.text_before_cursor,
                )
                completions.extend([(*x, rank) for x in aliases_m])

            elif suggestion["type"] == "database":
                dbs_m = self.find_matches(
                    word_before_cursor,
                    self.databases,
                    text_before_cursor=document.text_before_cursor,
                )
                completions.extend([(*x, rank) for x in dbs_m])

            elif suggestion["type"] == "keyword":
                keywords_m = self.find_matches(
                    word_before_cursor,
                    self.keywords,
                    casing=self.keyword_casing,
                    text_before_cursor=document.text_before_cursor,
                )
                completions.extend([(*x, rank) for x in keywords_m])

            elif suggestion["type"] == "show":
                show_items_m = self.find_matches(
                    word_before_cursor,
                    self.show_items,
                    start_only=False,
                    fuzzy=True,
                    casing=self.keyword_casing,
                    text_before_cursor=document.text_before_cursor,
                )
                completions.extend([(*x, rank) for x in show_items_m])

            elif suggestion["type"] == "change":
                change_items_m = self.find_matches(
                    word_before_cursor,
                    self.change_items,
                    start_only=False,
                    fuzzy=True,
                    text_before_cursor=document.text_before_cursor,
                )
                completions.extend([(*x, rank) for x in change_items_m])

            elif suggestion["type"] == "user":
                users_m = self.find_matches(
                    word_before_cursor,
                    self.users,
                    start_only=False,
                    fuzzy=True,
                    text_before_cursor=document.text_before_cursor,
                )
                completions.extend([(*x, rank) for x in users_m])

            elif suggestion["type"] == "special":
                special_m = list(
                    self.find_matches(
                        word_before_cursor,
                        self.special_commands,
                        start_only=True,
                        fuzzy=False,
                        text_before_cursor=document.text_before_cursor,
                    )
                )
                # specials are special, and go early in the candidates, first if possible
                completions.extend([(*x, 0) for x in special_m])
                special_command_candidates.update(x[0] for x in special_m)

            elif suggestion["type"] == "favoritequery":
                if hasattr(FavoriteQueries, 'instance') and hasattr(FavoriteQueries.instance, 'list'):
                    queries_m = self.find_matches(
                        word_before_cursor,
                        FavoriteQueries.instance.list(),
                        start_only=False,
                        fuzzy=True,
                        text_before_cursor=document.text_before_cursor,
                    )
                    completions.extend([(*x, rank) for x in queries_m])

            elif suggestion['type'] == 'favoritequery_template_key':
                if hasattr(FavoriteQueries, 'instance') and hasattr(FavoriteQueries.instance, 'get'):
                    query = FavoriteQueries.instance.get(suggestion['name'])
                    if query is not None:
                        try:
                            template_keys = find_favorite_query_template_keys(query)
                        except TemplateError:
                            continue
                        used_keys = suggestion['used_keys']
                        candidates = [
                            f'--{key}=' for key in sorted(template_keys - used_keys) if favorite_query_variable_pattern.fullmatch(key)
                        ]
                        keys_m = self.find_matches(
                            word_before_cursor,
                            candidates,
                            start_only=True,
                            fuzzy=False,
                            text_before_cursor=document.text_before_cursor,
                        )
                        completions.extend([(*x, rank) for x in keys_m])

            elif suggestion["type"] == "table_format":
                formats_m = self.find_matches(
                    word_before_cursor,
                    self.table_formats,
                    text_before_cursor=document.text_before_cursor,
                )
                completions.extend([(*x, rank) for x in formats_m])

            elif suggestion["type"] == "file_name":
                source_filename = suggestion.get('source_filename')
                if source_filename is None:
                    file_names_m = self.find_files(word_before_cursor)
                else:
                    source_file_completion_length = len(source_filename)
                    quote = source_filename[0] if source_filename[:1] in ("'", '"') else None
                    partial_path = source_filename[1:] if quote else source_filename
                    if quote and partial_path.endswith(quote):
                        partial_path = partial_path[:-1]
                    base_path, _last_path, _position = parse_path(partial_path)
                    file_names_m = (
                        (
                            self._quote_source_path(
                                os.path.join(base_path, path) if base_path and not path.startswith('~') else path,
                                quote,
                            ),
                            fuzziness,
                        )
                        for path, fuzziness in self.find_files(partial_path)
                    )
                completions.extend([(*x, rank) for x in file_names_m])
                # for filenames we _really_ want directories to go last
                rigid_sort = True
                length_based_on_path = True
            elif suggestion["type"] == "llm":
                if not word_before_cursor:
                    tokens = document.text.split()[1:]
                else:
                    tokens = document.text.split()[1:-1]
                possible_entries = llm.get_completions(tokens)
                subcommands_m = self.find_matches(
                    word_before_cursor,
                    possible_entries,
                    start_only=False,
                    fuzzy=True,
                    text_before_cursor=document.text_before_cursor,
                )
                completions.extend([(*x, rank) for x in subcommands_m])

            elif suggestion['type'] == 'special_subcommand':
                subcommands_m = self.find_matches(
                    word_before_cursor,
                    suggestion['subcommands'],
                    start_only=True,
                    fuzzy=False,
                    text_before_cursor=document.text_before_cursor,
                )
                completions.extend([(*x, rank) for x in subcommands_m])

            elif suggestion['type'] == 'config_property':
                prefix = suggestion['prefix']
                property_matches = self.find_fuzzy_matches(prefix, prefix.lower(), self.config_property_names)
                completions.extend([(*x, rank) for x in property_matches])
                config_property_length = len(prefix)
                completion_filter_text = prefix.lower()

            elif suggestion['type'] == 'dsn_alias':
                if hasattr(DsnAliases, 'instance'):
                    aliases_m = self.find_matches(
                        word_before_cursor,
                        DsnAliases.instance.list(),
                        start_only=True,
                        fuzzy=False,
                        text_before_cursor=document.text_before_cursor,
                    )
                    completions.extend([(*x, rank) for x in aliases_m])

            elif suggestion["type"] == "enum_value":
                enum_values = self.populate_enum_values(
                    suggestion["tables"],
                    suggestion["column"],
                    suggestion.get("parent"),
                )
                if enum_values:
                    quoted_values = [self._quote_sql_string(value) for value in enum_values]
                    completions = [
                        (*x, rank)
                        for x in self.find_matches(
                            word_before_cursor,
                            quoted_values,
                            text_before_cursor=document.text_before_cursor,
                        )
                    ]
                    break

        def completion_sort_key(item: tuple[str, int, int], text_for_len: str):
            candidate, fuzziness, rank = item
            candidate_frecency = frecency_score(candidate, frecency) if frecency else 0.0
            if not text_for_len:
                # Sort by the rank (the order of the completion type), then frecency.
                return (0, rank, -candidate_frecency, 0)
            elif candidate.lower().startswith(text_for_len):
                # Direct prefix matches are equally relevant; prefer frecency before length.
                return (0, 0, -candidate_frecency, -1000 + len(candidate))
            # Sort by fuzziness, rank, and frecency.
            # todo add alpha here, or original order?
            return (fuzziness, rank, -candidate_frecency, 0)

        if rigid_sort:
            uniq_completions_str = dict.fromkeys(x[0] for x in completions)
        else:
            sorted_completions = sorted(completions, key=lambda item: completion_sort_key(item, completion_filter_text))
            uniq_completions_str = dict.fromkeys(x[0] for x in sorted_completions)

        if config_property_length is not None:
            return (Completion(x, -config_property_length) for x in uniq_completions_str)
        elif source_file_completion_length is not None:
            return (Completion(x, -source_file_completion_length) for x in uniq_completions_str)
        elif length_based_on_path:
            return (
                Completion(
                    x,
                    -len(last_for_len_paths),
                    display=f'{x}{self.indexed_column_suffix}' if x in indexed_column_candidates else None,
                    display_meta=self.special_command_snippets.get(x) if x in special_command_candidates else None,
                    style=_INDEXED_COLUMN_STYLE if x in indexed_column_candidates else '',
                )
                for x in uniq_completions_str
            )
        else:
            return (
                Completion(
                    x,
                    -len(text_for_len),
                    display=f'{x}{self.indexed_column_suffix}' if x in indexed_column_candidates else None,
                    display_meta=self.special_command_snippets.get(x) if x in special_command_candidates else None,
                    style=_INDEXED_COLUMN_STYLE if x in indexed_column_candidates else '',
                )
                for x in uniq_completions_str
            )

    def find_files(self, word: str) -> Generator[tuple[str, int], None, None]:
        """Yield matching directory or file names.

        :param word:
        :return: iterable

        """
        # todo position is ignored, but may need to be used
        # todo fuzzy matches for filenames
        base_path, last_path, position = parse_path(word)
        paths = suggest_path(word)
        for name in paths:
            suggestion = complete_path(name, last_path)
            if suggestion:
                yield (suggestion, Fuzziness.PERFECT)

    @staticmethod
    def _quote_source_path(path: str, quote: str | None) -> str:
        is_directory = path.endswith(('/', os.sep))
        if is_directory and any(character.isspace() for character in path) and not path.startswith(('/', '~', './')):
            path = f'./{path}'
        if quote:
            return f'{quote}{path}' if is_directory else f'{quote}{path}{quote}'
        if not any(character.isspace() for character in path):
            return path
        if is_directory:
            return f'"{path}' if WIN else f"'{path}"
        return subprocess.list2cmdline([path]) if WIN else shlex.quote(path)

    def populate_scoped_cols(self, scoped_tbls: list[tuple[str | None, str, str | None]]) -> list[str]:
        """Find all columns in a set of scoped_tables
        :param scoped_tbls: list of (schema, table, alias) tuples
        :return: list of column names
        """
        columns = []
        meta = self.dbmetadata

        # if scoped tables is empty, this is just after a SELECT so we
        # show all columns for all tables in the schema.
        if len(scoped_tbls) == 0 and self.dbname:
            # ``dbname`` can point at a schema whose metadata has not been
            # loaded yet: a ``USE`` switch updates the live completer's
            # ``dbname`` immediately while the matching tables are still being
            # fetched on a background thread. Default to an empty mapping in
            # that window instead of raising ``KeyError``. Grab the per-schema
            # dict once so a concurrent refresh swapping it out cannot break
            # the iteration mid-loop.
            schema_tables = meta["tables"].get(self.dbname, {})
            for table in schema_tables:
                columns.extend(schema_tables[table])
            return columns or ['*']

        # query includes tables, so use those to populate columns
        for tbl in scoped_tbls:
            # A fully qualified schema.relname reference or default_schema
            # DO NOT escape schema names.
            schema = tbl[0] or self.dbname
            relname = tbl[1]
            escaped_relname = self.escape_name(tbl[1])

            # We don't know if schema.relname is a table or view. Since
            # tables and views cannot share the same name, we can check one
            # at a time
            try:
                columns.extend(meta["tables"][schema][relname])

                # Table exists, so don't bother checking for a view
                continue
            except KeyError:
                try:
                    columns.extend(meta["tables"][schema][escaped_relname])
                    # Table exists, so don't bother checking for a view
                    continue
                except KeyError:
                    pass

            try:
                columns.extend(meta["views"][schema][relname])
            except KeyError:
                pass

        return columns

    def populate_scoped_indexed_columns(self, scoped_tbls: list[tuple[str | None, str, str | None]]) -> set[str]:
        """Find leading indexed columns in a set of scoped tables."""
        metadata = self.dbmetadata["indexed_columns"]
        indexed_columns: set[str] = set()

        if not scoped_tbls:
            for columns in metadata.get(self.dbname, {}).values():
                indexed_columns.update(columns)
            return indexed_columns

        for schema, relname, _alias in scoped_tbls:
            schema_meta = metadata.get(schema or self.dbname, {})
            escaped_relname = self.escape_name(relname)
            indexed_columns.update(schema_meta.get(relname, set()))
            indexed_columns.update(schema_meta.get(escaped_relname, set()))

        return indexed_columns

    def populate_enum_values(
        self,
        scoped_tbls: list[tuple[str | None, str, str | None]],
        column: str,
        parent: str | None = None,
    ) -> list[str]:
        values: list[str] = []
        meta = self.dbmetadata["enum_values"]
        column_key = self._escape_identifier(column)
        parent_key = self._strip_backticks(parent) if parent else None

        for schema, relname, alias in scoped_tbls:
            if parent_key and not self._matches_parent(parent_key, schema, relname, alias):
                continue

            schema = schema or self.dbname
            table_meta = meta.get(schema, {})
            escaped_relname = self.escape_name(relname)

            for rel_key in {relname, escaped_relname}:
                columns = table_meta.get(rel_key)
                if columns and column_key in columns:
                    values.extend(columns[column_key])

        return list(dict.fromkeys(values))

    def _escape_identifier(self, name: str) -> str:
        return self.escape_name(self._strip_backticks(name))

    @staticmethod
    def _strip_backticks(name: str | None) -> str:
        if name and name[0] == "`" and name[-1] == "`":
            return name[1:-1]
        return name or ""

    @staticmethod
    def _matches_parent(parent: str, schema: str | None, relname: str, alias: str | None) -> bool:
        if alias and parent == alias:
            return True
        if parent == relname:
            return True
        if schema and parent == f"{schema}.{relname}":
            return True
        return False

    @staticmethod
    def _quote_sql_string(value: str) -> str:
        return "'" + value.replace("'", "''") + "'"

    def populate_schema_objects(self, schema: str | None, obj_type: str, columns: list[str] | None = None) -> list[str]:
        """Returns list of tables or functions for a (optional) schema"""
        metadata = self.dbmetadata[obj_type]
        schema = schema or self.dbname
        try:
            objects = list(metadata[schema].keys())
        except KeyError:
            # schema doesn't exist
            objects = []

        filtered_objects: list[str] = []
        remaining_objects: list[str] = []

        # If the requested object type is tables and the user already entered
        # columns, return a filtered list of tables (or views) that contain
        # one or more of the given columns. If a table does not contain the
        # given columns, add it to a separate list to add to the end of the
        # filtered suggestions.
        if obj_type == "tables" and columns and objects:
            for obj in objects:
                matched = False
                for column in metadata[schema][obj]:
                    if column in columns:
                        filtered_objects.append(obj)
                        matched = True
                        break
                if not matched:
                    remaining_objects.append(obj)
        else:
            filtered_objects = objects
        return filtered_objects + remaining_objects
