#!/usr/bin/env python3
# SPDX-License-Identifier: BSD-2-Clause

# Massage a JSON traits table into an equivalent Go initializer.
# We do this in code generation, rather than reading in JSON
# at loccount startup, to make it faster.
#
# These options:
#
# -c      match specified regexp across linecountered
# -g arg  Generate a Go initializer
# -s      Statistics on language types
# -t      Compare known counter tables
# -T      Markdown table of known counter language counts
# -l tag  List entries with specified quirk
# -v      Enable verbose debugging

# ruff: noqa E731

import json, sys, getopt, copy, re
from typing import Any, BinaryIO, Final, List, Set, Dict, Tuple, BinaryIO, Callable, Set
from pprint import pformat
from io import StringIO

# You need ruamel.yaml for the surgeon function,
# but not otherwise. It's not used in a normal build.
try:
    from ruamel.yaml import YAML

    have_ruamel = True
    yaml = YAML(typ="rt")
    yaml.preserve_quotes = True
    # Disable line wrapping in the output
    # Without this setting, the YAML dumper's line-wrapping code may clobber
    # tokens in long string lists like Require/Forbid declarations because it doesn't
    # know that string literals should be immutable.
    yaml.width = 10**9
    yaml.safe_load = yaml.load
except ImportError:
    import yaml

    have_ruamel = False

JSONDict = dict[str, Any]

verbose = False

classes: dict[str, str] = {}


def split_mask_flags(raw: Any) -> list[str]:
    if isinstance(raw, str):
        return [piece for piece in raw.split("|") if piece]
    if isinstance(raw, list):
        out: list[str] = []
        for item in raw:
            if not isinstance(item, str):
                raise ValueError(f"flag must be string, got {item!r}")
            out.extend(piece for piece in item.split("|") if piece)
        return out
    if raw is None:
        return []
    raise ValueError(f"flag mask must be string or list, got {raw!r}")


def join_mask_flags(flags: list[str]) -> str:
    seen: set[str] = set()
    out: list[str] = []
    for flag in flags:
        if flag not in seen:
            seen.add(flag)
            out.append(flag)
    return "|".join(out)


language_metadata_keys: Final[Set[str]] = {
    "aka",
    "description",
    "links",
    "base",
    "variants",
}
legacy_language_metadata_keys: Final[Set[str]] = {"aka", "description", "links"}


def normalize_links(raw: Any, lang_name: str) -> dict[str, str]:
    if raw is None:
        return {}
    if not isinstance(raw, dict):
        raise ValueError(f"language {lang_name!r} has invalid links value {raw!r}")
    links: dict[str, str] = {}
    for label, url in raw.items():
        if not isinstance(label, str) or not isinstance(url, str):
            raise ValueError(
                f"language {lang_name!r} has non-string links entry {label!r}: {url!r}"
            )
        links[label] = url
    return links


def structured_language_parts(
    lang_name: str, lang_block: dict[str, Any]
) -> tuple[dict[str, Any] | None, list[tuple[str, dict[str, Any]]]]:
    """
    Normalize a language block into an optional base mapping plus a list of
    variant mappings.

    The preferred multivariant schema is:

        Language:
          description: ...
          base: {...shared traits...}
          variants:
            first: {...matcher...}
            second: {...matcher...}

    For compatibility, single-variant and legacy multivariant entries may
    still place variant mappings directly under the language block.
    """
    if "variants" in lang_block:
        raw_base = lang_block.get("base", {})
        if raw_base is None:
            base = {}
        elif isinstance(raw_base, dict):
            base = raw_base
        else:
            raise ValueError(f"language {lang_name!r} has non-mapping base")

        raw_variants = lang_block["variants"]
        if not isinstance(raw_variants, dict):
            raise ValueError(f"language {lang_name!r} variants is not a mapping")

        legacy_items = [
            (k, v) for k, v in lang_block.items() if k not in language_metadata_keys
        ]
        if legacy_items:
            stray = ", ".join(repr(k) for k, _ in legacy_items)
            raise ValueError(
                f"language {lang_name!r} mixes variants: with legacy entries {stray}"
            )

        variant_items = list(raw_variants.items())
        if not variant_items:
            raise ValueError(f"language {lang_name!r} has no variants")
        return base, variant_items

    if "base" in lang_block:
        raise ValueError(f"language {lang_name!r} has base without variants")

    variant_items = [
        (k, v) for k, v in lang_block.items() if k not in legacy_language_metadata_keys
    ]
    if not variant_items:
        raise ValueError(f"language {lang_name!r} has no variants")
    return None, variant_items


def surgeon(
    source_path: str,
    hook: Callable[[str, str, str, dict[str, Any]], None] | None,
    sink: BinaryIO,
) -> None:
    """
    Read a languages.yaml file, apply a mutating hook to each structured base
    or variant mapping, and write the modified YAML to a binary output sink.
    """
    if not have_ruamel:
        raise RuntimeError("surgeon mode requires ruamel.yaml")

    with open(source_path, "r", encoding="utf-8") as fp:
        root = yaml.safe_load(fp)

    if root is None:
        root = {}

    if not isinstance(root, dict):
        raise ValueError("top-level YAML document is not a mapping")

    languages = root["languages"]

    for name, lang_node in languages.items():
        if not isinstance(lang_node, dict):
            raise ValueError(f"language {name!r} does not map to a mapping")

        description = lang_node.get("description", "")
        if description is None:
            description = ""
        else:
            description = str(description)

        if hook is not None:
            base, variant_items = structured_language_parts(name, lang_node)
            if "base" in lang_node and isinstance(lang_node["base"], dict):
                hook(name, description, "base", lang_node["base"])
            for variant, record in variant_items:
                if not isinstance(record, dict):
                    raise ValueError(
                        f"language {name!r} variant {variant!r} is not a mapping"
                    )
                hook(name, description, variant, record)

    buf = StringIO()
    yaml.dump(root, buf)
    sink.write(buf.getvalue().encode("utf-8"))


proprietary_only = {
    "8th",
    "Apex",
    "Apex Class",
    "AutoIt",  # Windows-only prpprietary freeware
    "Bitbucket Pipeline",
    "BizTalk Orchestration",
    "BizTalk Pipeline",
    "BrightScript",
    "C# Designer",
    "DOORS DXL",
    "Focus",
    "GAMS",
    "HiCAD",
    "Igor Pro",
    "Informix 4GL",
    "InstallShield",
    "Magik",
    "Max",
    "Metal",
    "MQL4",
    "MQL5",
    "MQL Header",
    "NASTRAN DMAP",
    "Oracle Forms",
    "Oracle Reports",
    "Patran Command Language",
    "PL/SQL",
    "PODL",
    "PowerBuilder",
    "PRACTICE",
    "Q",
    "ShaderLab",
    "SKILL",
    "SKILL++",
    "SPL",
    "SQF",
    "Stata",
    "Teamcenter MET",
    "Teamcenter MTH",
    "TNSDL",
    "Unity Prefab",
    "Unreal Developer Markdown",
    "Unreal Project",
    "Unreal Plugin",
    "Unreal Script",
    "Unreal Shader",
    "Unreal Shader Header",
    "USS",
    "UXML",
    "VB6",
    "VBA",
    "Visualforce Component",
    "Visualforce Page",
    "Visual Studio Solution",
    "Visual Studio Project",
    "Windows Message",
    "Windows Module Definition",
    "Windows Resource Script",
    "WXML",
    "WXSS",
    "Xcode Config",
    "X++",
}


toy_languages = {
    "Alchemist",  # *
    "Brainfuck",
    "EmiT",
    "Emojicode",
    "Headache",
    "Hex0",
    "Hex1",
    "Hex2",
    "HolyC",
    "LOLCODE",
    "Luna",  # Dead since 2016
    "Mlatu",
    "UMPL",
    "Wenyan",
}


exceptions_and_mixed = {
    "Alore",  # Language site is dead
    "Astro (WIP)",  # Not the web framework; astrolang/astro is WIP
    "BASIC-256",  # BASIC from a dead micro
    "Beads",  # A Java library that Rosetta Code lists in error
    "CCScript",  # Niche language for SNES hacking
    "DenizenScript",  # Open source, but Minecraft only
    "DIET",  # Site is dead
    "FEN",  # Chess notation? Really?
    "ForgeConfig",  # Only used in Minecraft modding
    "Juniper Junos",  # Tries to grab .conf
    "Koka",  # Research language, WIP, unstable
    "Monkey C",  # Open source but tied to a single proprietary device
    "Nu",  # Moribund - no activity in 3 years, no release since 2019.
    "Phoenix LiveView",  # Doesn't exist - it's the runtime for EEx
    "Polly",  # Obscure abandoned Rust templating language
    "Red",  # Toy - still generating 32-bit binaries in 2026.
    "Redscript",  # Open source, but single-platform game scripting language
    "Softbridge Basic",  # Proprietary and dead
    "Tiger",  # Teaching language used in one course
    "Up",  # Experimental interpreter language
}

kill_list = proprietary_only | toy_languages | exceptions_and_mixed

# Common synonyms to loccount names.
common_synonyms = {
    "Apache Thrift": "Thrift",
    "Arduino Sketch": "Arduino C++",
    "Assembly": "asm",
    "AutoHotKey": "AutoHotkey",
    "Awk": "AWK",
    "BASH": "bash",
    "Batch": "batchfile",
    "Bourne Shell": "bash",
    "Cucumber": "Gherkin",
    "C-Header": "C Header",
    "C++-Header": "C++ Header",
    "Cap'n Proto": "Capnp",
    "C Shell": "csh",
    "ColdFusion": "CFML",
    "ColdFusion CFScript": "CFScript",
    "Coq": "Rocq",
    "Device Tree": "DeviceTree",
    "Fortran Legacy": "Fortran77",
    "FORTRAN Legacy": "Fortran77",
    "Fortran Modern": "Fortran90",
    "FORTRAN Modern": "Fortran90",
    "Fish": "fish",
    "JAI": "Jai",
    "Jinja Templates": "Jinja",
    "Jupyter Notebook": "Jupyter",
    "LD Script": "LinkerScript",
    "Lilypond": "LilyPond",
    "LESS": "Less",
    "LISP": "Lisp",
    "Lua": "lua",
    "M4": "m4",
    "MSBuild script": "MSBuild",
    "Objective-C": "Objective C",
    "Objective-C++": "Objective C++",
    "ooRexx": "Rexx",
    "Plain Text": "text",
    "Plan9 Shell": "rc",
    "Protocol Buffers": "Protobuf",
    "Qt Project": "qmake",
    "Reason": "ReasonML",
    "ReStructuredText": "reStructuredText",
    "Standard ML": "SML",
    "TLA": "TLA+",
    "Ruby HTML": "ERB",
    "Tcl/Tk": "Tcl",
    "Unity-Prefab": "Unity Prefab",
    "VimL": "Vim script",
    "VimScript": "Vim script",
    "Visual-Basic": "Visual Basic",
    "Web Services Description Language": "WSDL",
    "Yang": "YANG",
    "Zsh": "zsh",
    "hoon": "Hoon",
    "lex": "Lex",
}

#
# How to build a class encapsulatin for another line-counters database.
#
# 1. Grab a copy, whrgwe JSON or some lit in a source file
#    Stash it in tests. Update it when you notive they've shipped.
#
# 2. Write a class init that reads in the stashed file and parses it
#    into a list of dict records.
#
# 3. As the initializer runs, it should add a loccount_name to each
#    record. Important! This can be the native name initially --
#    you're going to build a mapping from native names to loccount ones
#    over time, and it can initially be empty.
#
# 4. Write a records() method that returns the parsed data, with the
#    loccount_name member in it.
#
# 5. Write a loccount_set() member that returns a list of language
#    names. None, some, or all can be mapped to loccount names.


class Loccount:
    def __init__(self, resolve: bool = True):
        parts: dict[str, JSONDict] = {}
        with open("languages.yaml", encoding="utf-8") as fp:
            self._yaml = yaml.safe_load(fp)

        if self._yaml is None:
            raise ValueError()

        languages = self._yaml["languages"]

        # Ugh...
        global classes
        classes = self._yaml["classes"]

        self._data: list[Dict[str, Any]] = []
        self._refindex: Dict[str, Dict[str, Any]] = {}

        # Walk through the YAML tree, accumulating variant records for languages.
        for lang_name, lang_block in languages.items():
            if not isinstance(lang_block, dict):
                raise ValueError(f"language {lang_name!r} does not map to a mapping")

            description = lang_block.get("description", "")
            if description is None:
                description = ""
            links = normalize_links(lang_block.get("links", {}), lang_name)
            raw_aka = lang_block.get("aka", [])
            if raw_aka is None:
                raw_aka = []
            if isinstance(raw_aka, str):
                aka = raw_aka
            elif isinstance(raw_aka, list):
                aka = "\n".join(str(item) for item in raw_aka)
            else:
                raise ValueError(
                    f"language {lang_name!r} has invalid aka value {raw_aka!r}"
                )
            self._refindex[lang_name] = {
                "description": str(description),
                "aka": aka,
                "links": links,
                "index": -1,
                "variants": 0,
            }

            base_body, variant_items = structured_language_parts(lang_name, lang_block)
            self._refindex[lang_name]["variants"] = len(variant_items)

            if base_body is not None:
                base_entry = copy.deepcopy(base_body)
                base_entry["Name"] = lang_name
                base_entry["Var"] = "base"
                parts[lang_name] = base_entry

            for variant_name, variant_body in variant_items:
                if not isinstance(variant_body, dict):
                    raise ValueError(
                        f"language {lang_name!r} variant {variant_name!r} is not a mapping"
                    )

                if base_body is None:
                    entry = dict(variant_body)
                else:
                    entry = copy.deepcopy(base_body)
                    entry.update(copy.deepcopy(variant_body))
                entry["Name"] = lang_name
                entry["Var"] = variant_name

                for ext in entry.get("Ext", []):
                    if not isinstance(ext, str) or not ext.startswith("."):
                        raise ValueError(
                            f"language {lang_name!r} variant {variant_name!r} has invalid Ext entry {ext!r}: entries must start with '.'"
                        )

                for filename in entry.get("Files", []):
                    if not isinstance(filename, str) or "/" in filename:
                        raise ValueError(
                            f"language {lang_name!r} variant {variant_name!r} has invalid Files entry {filename!r}: entries must be basenames without '/'"
                        )

                # Legacy multivariant entries still resolve Like against the
                # last variant seen. Structured base/variants entries avoid
                # that ambiguity by pinning parts[lang_name] to base above.
                if base_body is None:
                    parts[lang_name] = entry

                if self._refindex[lang_name]["index"] < 0:
                    self._refindex[lang_name]["index"] = len(self._data)

                self._data.append(entry)

        # Like resolution
        if resolve:
            for i in range(len(self._data)):
                # Warning: This could infinite-loop if there are
                # ever circular inheritance chains. Can't happen
                # yet, as Like targets nust precede their users.
                # If that changers, add a cycle detector here.
                entry = self._data[i]
                while "Like" in entry:
                    try:
                        proto = parts[entry["Like"]]
                    except KeyError:
                        print(
                            f'{entry["Name"]} is missing target of Like {entry["Like"]}'
                        )
                        raise SystemExit(1)
                    try:
                        for k in proto:
                            if k in {"Name", "Ext", "Files"}:
                                continue
                            elif k == "Fl":
                                entry[k] = join_mask_flags(
                                    split_mask_flags(proto[k])
                                    + split_mask_flags(entry.get(k, ""))
                                )
                            elif isinstance(proto[k], list):
                                # List inheritence has to be prepending ratrher than appending,
                                # otherwise logic that scand sor delimiter pairs will stop at the
                                # partent's version rgatner than the chiild's.
                                entry[k] = entry.get(k, []) + proto[k]
                            elif k not in entry:
                                entry[k] = proto[k]
                    except TypeError:
                        print(
                            f'{entry["Name"]} internal error inheriting from {entry["Like"]}'
                        )
                        raise SystemExit(1)
                    del entry["Like"]
                self._data[i] = entry

        # We build extension cliques late so Require/Forbid can be inherited.
        if resolve:
            extension_cliques: dict[str, list[dict[str, Any]]] = {}
            for entry in self._data:
                if "Ext" in entry:
                    has_forbids = bool(entry.get("Forbid"))
                    has_requires = bool(entry.get("Require"))
                    for ext in entry["Ext"]:
                        if ext not in extension_cliques:
                            extension_cliques[ext] = []
                        extension_cliques[ext].append(
                            {
                                "name": entry.get("Name", "(unnamed)"),
                                "has_forbids": has_forbids,
                                "has_requires": has_requires,
                            }
                        )
            # Make sure there is only one fallback per file-extension clique.
            sanity_failures = []
            for ext, clique in extension_cliques.items():
                if len(clique) <= 1:
                    continue
                for i, member in enumerate(clique):
                    if member["has_forbids"] or member["has_requires"]:
                        continue
                    if i == len(clique) - 1:
                        continue
                    sanity_failures.append(
                        f'Extension {ext} entry {member["name"]} '
                        + "has neither of Require or Forbid in non-fallback position"
                    )
            if sanity_failures:
                for failure in sanity_failures:
                    print(failure, file=sys.stderr)
                raise SystemExit(1)

        self._names = {
            entry["Name"] for entry in self._data if "Ext" in entry or "Files" in entry
        }

    @staticmethod
    def _go_quote(s: str) -> str:
        return (
            '"'
            + (
                s.replace("\\", "\\\\")
                .replace('"', '\\"')
                .replace("\n", "\\n")
                .replace("\t", "\\t")
                .replace("\r", "\\r")
            )
            + '"'
        )

    def name(self):
        return self.__class__.__name__

    def records(self) -> Tuple[Dict]:
        "A list of dictionaries made from resolved records. Name members may not be unique."
        return tuple(self._data)

    def loccount_set(self) -> Set[str]:
        "Names, excludoing '-like' entries without extensions or files"
        return self._names

    def go_dump(self, wfp: BinaryIO) -> None:

        def _camelize(name: str) -> str:
            """
            Turn a traits/YAML member name into an exported Go field name.
            Keeps existing CamelCase names intact enough for this use case.
            """
            if not name:
                return name
            return name[0].upper() + name[1:]

        def _infer_go_type(field_name, value: Any) -> str:
            if field_name in {"Bc", "Str"}:
                if not isinstance(value, list):
                    raise ValueError(
                        f"{field_name}: expected list, got {type(value).__name__}"
                    )
                return "[]delimSpec"
            if isinstance(value, str):
                return "string"
            if isinstance(value, list):
                if not value:
                    raise ValueError("cannot infer Go type from empty list alone")
                first = value[0]
                if isinstance(first, str):
                    if not all(isinstance(x, str) for x in value):
                        raise ValueError(
                            f"{name}: mixed list element types in {value!r}"
                        )
                    return "[]string"
                if isinstance(first, (list, tuple)):
                    if not all(
                        isinstance(x, (list, tuple))
                        and isinstance(x[0], str)
                        and isinstance(x[1], str)
                        for x in value
                    ):
                        raise ValueError(f"unsupported nested list shape in {value!r}")
                    return "[][]string"
            raise ValueError(f"unsupported value type for {value!r}")

        def _merge_go_types(name: str, old: str | None, new: str) -> str:
            if old is None:
                return new
            if old != new:
                raise ValueError(
                    f"{name}: inconsistent Go types: saw both {old} and {new}"
                )
            return old

        def _emit_go_value(value: Any, go_type: str) -> str:
            def _split_pair_flags(raw: Any) -> list[str]:
                if not isinstance(raw, str):
                    raise ValueError(f"delimiter flag must be string, got {raw!r}")
                return [piece for piece in raw.split("|") if piece]

            if go_type == "string":
                return Loccount._go_quote(value)

            if go_type == "[]string":
                assert isinstance(value, list)
                return (
                    "[]string{"
                    + ", ".join(Loccount._go_quote(str(x)) for x in value)
                    + "}"
                )

            if go_type == "[][]string":
                assert isinstance(value, list)
                parts = []
                for sublist in value:
                    parts.append(
                        "[]string{"
                        + ",".join(Loccount._go_quote(str(p)) for p in sublist)
                        + "}"
                    )
                return "[][]string{" + ", ".join(parts) + "}"

            if go_type == "[]delimSpec":
                assert isinstance(value, list)
                parts = []
                for pair in value:
                    if not isinstance(pair, (list, tuple)) or len(pair) < 2:
                        raise ValueError(f"unsupported delimiter shape in {value!r}")
                    if not isinstance(pair[0], str) or not isinstance(pair[1], str):
                        raise ValueError(
                            f"delimiter opener/closer must be strings in {pair!r}"
                        )
                    flags: list[str] = []
                    for raw_flag in pair[2:]:
                        flags.extend(_split_pair_flags(raw_flag))
                    flag_expr = "|".join(flags) if flags else "0"
                    parts.append(
                        "delimSpec{"
                        + f"Open: {Loccount._go_quote(pair[0])}, "
                        + f"Close: {Loccount._go_quote(pair[1])}, "
                        + f"Flags: {flag_expr}"
                        + "}"
                    )
                return "[]delimSpec{" + ", ".join(parts) + "}"

            if go_type == "uint":
                flags = split_mask_flags(value)
                return "|".join(flags)

            raise ValueError(f"unsupported Go type {go_type}")

        wfp.write(
            b"""\
// This file is generated - do not hand-hack!
package main

"""
        )

        # Make non-languages top-level dictionaries available.
        # Fail hard if the YAML has stray top-level scalars or invalid names;
        # otherwise we can emit partial Go before discovering the problem.
        for item_name, item_value in self._yaml.items():
            if item_name == "languages":
                continue
            if not isinstance(item_name, str) or not re.match(
                r"^[A-Za-z_][A-Za-z0-9_]*$", item_name
            ):
                raise ValueError(f"invalid top-level name {item_name!r}")
            if not hasattr(item_value, "items"):
                raise ValueError(
                    f"top-level item {item_name!r} must be a mapping, got {type(item_value).__name__}"
                )
            wfp.write(b"var " + item_name.encode("utf-8") + b" = map[int]string{\n")
            try:
                for key, value in item_value.items():
                    wfp.write(
                        f"\t{key}: {Loccount._go_quote(value)},\n".encode("utf-8")
                    )
            except AttributeError as e:
                sys.stderr.write("f{key}: %{e}\n")
            wfp.write(b"}\n")

        entries = self.records()
        references = self._refindex

        flagify: Final[Set[str]] = {"Fl"}

        # Track member names in first-seen order across variants.
        member_order: list[str] = []
        member_types: dict[str, str] = {}

        for record in entries:
            for key, value in record.items():
                if key not in member_types:
                    member_order.append(key)
                    member_types[key] = None  # type: ignore[assignment]

                if value is None:
                    continue

                if key in flagify:
                    member_types[key] = "uint"
                    continue

                field_name = key
                inferred = _infer_go_type(field_name, value)
                member_types[key] = _merge_go_types(
                    field_name, member_types[key], inferred
                )

        # Default any never-populated member type if that somehow occurred.
        for key in member_order:
            if member_types[key] is None:
                raise ValueError(f"could not infer Go type for field {key!r}")

        # This blocl produces a Go initializer and accumukates the set of flags.
        out = []
        out.append("type genericLanguage struct {")
        for key in member_order:
            go_field = _camelize(key)
            go_type = member_types[key]
            out.append(f"\t{go_field} {go_type}")
        out.append("}")
        out.append("")

        flag_values = set(["cnest", "ml", "bse", "sol", "re", "outdent"])
        out.append("    var genericLanguages = []genericLanguage{")
        for record in entries:
            out.append("\t{")
            for key in member_order:
                if key not in record:
                    continue
                go_field = _camelize(key)
                go_type = member_types[key]
                go_value = _emit_go_value(record[key], go_type)
                if key in {"Bc", "Str"}:
                    for pair in record[key]:
                        for raw_flag in pair[2:]:
                            flag_values |= set(
                                [piece for piece in raw_flag.split("|") if piece]
                            )
                if key in flagify:
                    pieces = [piece for piece in go_value.split("|") if piece]
                    flag_values |= set(pieces)
                    go_value = "|".join(pieces)
                    if not go_value:
                        continue
                out.append(f"\t\t{go_field}: {go_value},")
            out.append("\t},")
        out.append("}")
        out.append("")

        wfp.write(
            b"""\
// Syntax quirk flags.  Those with an x prefix don't currently affect parsing,
// they're declared to support things we might want to do in the future.
const (
"""
        )
        flag_values = list(flag_values)
        flag_values.sort()
        wfp.write(f"\t {flag_values[0]} = 1 << iota\n".encode("utf-8"))
        for flag in flag_values[1:]:
            wfp.write(f"\t {flag}\n".encode("utf-8"))
        wfp.write(
            b"""\
)

var flagLabels = []struct {
        flag  uint
        label string
}{
"""
        )
        for flag in flag_values:
            wfp.write(('\t {%s, "%s"},\n' % (flag, flag)).encode("utf-8"))
        wfp.write(b"}\n")

        wfp.write("\n".join(out).encode("utf-8") + b"\n")

        if references:
            wfp.write(
                b"type languageReference struct {\n\tdescription string\n\taka string\n\tlinks map[string]string\n\tindex int\n\tvariants int\n}\n"
            )
            wfp.write(b"var references = map[string]languageReference{\n")
            for key, ref in references.items():
                if ref["links"]:
                    link_parts = []
                    for label, url in ref["links"].items():
                        link_parts.append(
                            f"{Loccount._go_quote(label)}: {Loccount._go_quote(url)}"
                        )
                    links = "map[string]string{" + ", ".join(link_parts) + "}"
                else:
                    links = "map[string]string{}"
                content = (
                    f"    {Loccount._go_quote(key)}: "
                    + "languageReference{"
                    + f"description: {Loccount._go_quote(ref['description'])}, "
                    + f"aka: {Loccount._go_quote(ref['aka'])}, "
                    + f"links: {links}, "
                    + f"index: {ref['index']}, "
                    + f"variants: {ref['variants']}"
                    + "},\n"
                )
                wfp.write(content.encode("utf-8"))
            wfp.write(b"}\n")


class Tokei:
    # Map loccount names to Tokei IDs
    to_loccount: Final[dict[str, str]] = {
        "Abap": "ABAP",
        "Alex": "Alex (Eclipse)",
        "Apl": "APL",
        "Arduino": "Arduino C++",
        "Asn1": "ASN.1",
        "Asp": "ASP",
        "AspNet": "ASP.NET",
        "AspNet": "ASP.NET",
        "Assembly": "asm",  # *
        "AssemblyGAS": "asm",  # *
        "Ats": "ATS",
        "AutoHotKey": "AutoHotkey",
        "Autoit": "AutoIt",
        "AvaloniaXaml": "AXAML",  # *
        "Ba256": "BASIC-256",
        "Bash": "bash",
        "Batch": "batchfile",  # *
        "Bean": "Beancount",
        "Bitbake": "BitBake",
        "Bqn": "BQN",
        "CHeader": "C Header",
        "CSharp": "C#",
        "CShell": "csh",
        "Cil": "CIL",
        "Cobol": "COBOL",
        "ColdFusion": "CFML",
        "ColdFusionScript": "CFScript",
        "Coq": "Rocq",
        "Cpp": "C++",
        "CppHeader": "C++ Header",
        "Css": "CSS",
        "Cuda": "CUDA",
        "Cue": "CUE",
        "Daml": "DAML",
        "Docker": "Dockerfile",
        "DotNetResource": ".NET Resource",
        "DreamMaker": "DM",
        "Ebuild": "ebuild",
        "Edn": "EDN",
        "Eighth": "8th",
        "Elisp": "elisp",
        "Fish": "fish",
        "FreeMarker": "FTL",
        "FSharp": "F#",
        "FortranLegacy": "Fortran77",
        "FortranModern": "Fortran90",
        "Fstar": "F*",
        "GdScript": "GDScript",
        "GdShader": "GDShader",
        "GlimmerJs": "GlimmerJS",
        "GlimmerTs": "GlimmerTS",
        "Glsl": "GLSL",
        "Gml": "GML",
        "Gohtml": "GoTemplate",
        "Graphql": "GraphQL",
        "Hcl": "HCL",
        "Hex": "HEX",
        "HiCad": "HiCAD",
        "Hledger": "hledger",  # *
        "Hlsl": "HLSL",
        "Html": "HTML",
        "Ini": "INI",
        "IntelHex": "Intel HEX",
        "Jinja2": "Jinja",  # *
        "Jq": "jq",
        "Json": "JSON",
        "Jsx": "JSX",
        "Ksh": "ksh",
        "KvLanguage": "KVLang",
        "LLVM": "LLVM IR",
        "Lalrpop": "LALRPOP",
        "LolCode": "LOLCODE",
        "Lua": "lua",
        "M1Assembly": "asm",  # *
        "M4": "m4",
        "Mdx": "MDX",
        "ModuleDef": "Module-Definition",
        "MonkeyC": "Monkey C",
        "MsBuild": "MSBuild",
        "NuGetConfig": "NuGet Config",
        "ObjectiveC": "Objective C",
        "ObjectiveCpp": "Objective C++",
        "OpenPolicyAgent": "Rego",  # *
        "OpenQasm": "OpenQASM",
        "OpenScad": "OpenSCAD",
        "Php": "PHP",
        "PlantUml": "PlantUML",
        "Po": "PO",
        "PostCss": "PostCSS",
        "Qcl": "QCL",
        "Qml": "QML",
        "ReStructuredText": "reStructuredText",
        "Renpy": "Ren'Py",
        "RubyHtml": "ERB",
        "Scons": "SCons",
        "Sh": "sh",
        "Sml": "SML",
        "SpecmanE": "Specman e",
        "Sqf": "SQF",
        "Sqf": "SQF",
        "Sql": "SQL",
        "Svg": "SVG",
        "Swig": "SWIG",
        "Text": "text",
        "Tex": "TeX",
        "Toml": "TOML",
        "Tsx": "TSX",
        "Ttcn": "TTCN-3",
        "Unity-Prefab": "Unity Prefab",
        "UnrealDeveloperMarkdown": "Unreal Developer Markdown",
        "UnrealPlugin": "Unreal Plugin",
        "UnrealProject": "Unreal Project",
        "UnrealScript": "Unreal Script",
        "UnrealShader": "Unreal Shader",
        "UnrealShaderHeader": "Unreal Shader Header",
        "UrWeb": "Ur/Web",
        "UrWebProject": "Ur/Web Project",
        "VerilogArgsFile": "Verilog Args File",
        "Vhdl": "VHDL",
        "VimScript": "Vim script",
        "VisualBasic": "Visual Basic",
        "VisualStudioProject": "Visual Studio Project",
        "VisualStudioSolution": "Visual Studio Solution",
        "WenYan": "Wenyan",
        "Wix": "WiX",
        "Mathematica": "Wolfram",  # *
        "Xaml": "XAML",
        "XcodeConfig": "Xcode Config",
        "Xml": "XML",
        "Yaml": "YAML",
        "Zokrates": "ZoKrates",
        "asiidoc": "AsciiDoc",
        "cabal": "Cabal",
        "cmake": "CMake",
        "nu": "Nu",
        "org": "Org",
        "Zsh": "zsh",
    }

    def __init__(self):
        with open("tests/tokei.json", "r", encoding="utf-8") as rfp:
            tokei_obj = json.load(rfp)
        self._data = tokei_obj.get("languages")
        self._names = set()
        if not isinstance(self._data, dict):
            print(f"Tokei: missing or invalid languages map", file=sys.stderr)
            raise SystemExit(1)
        self._names = set()
        for k in self._data:
            aka = Tokei.to_loccount.get(k, k)
            self._data[k]["loccount"] = aka
            self._names.add(aka)

    def name(self):
        return self.__class__.__name__

    def records(self) -> Tuple[Dict]:
        "Dictionary of records, with an added 'loccount' member in each record that's the loccount name."
        return tuple({"Name": k, **v} for k, v in self._data.items())

    def loccount_set(self) -> Set[str]:
        return self._names


class Scc:
    to_loccount = {
        # Until the next comment, the right-hand forms are pre preferred by maintainnrs
        "Alex": "Alex (Haskell)",
        "Android Interface Definition Language": "AIDL",
        "ArkTs": "ArkTS",
        "AutoHotKey": "AutoHotkey",
        "Basic": "BASIC",
        "Bean": "Beancount",
        "Batch": "batchfile",
        "Bitbake": "BitBake",
        "Blade template": "Blade",
        "C Shell": "csh",
        "Cap'n Proto": "Capnp",
        "ColdFusion CFScript": "CFScript",
        "ColdFusion": "CFML",
        "Coq": "Rocq",
        "Cuda": "CUDA",
        "Document Type Definition": "DTD",
        "Elixir Template": "EEx",
        "Emacs Lisp": "elisp",
        "Fish": "fish",
        "Fragment Shader File": "GLSL Fragment Shader File",
        "Freemarker Template": "FTL",
        "Game Maker Language": "GML",
        "Gherkin Specification": "Gherkin",
        "HAML": "Haml",
        "JAI": "Jai",
        "JavaServer Pages": "JSP",
        "Korn Shell": "ksh",
        "LD Script": "LinkerScript",
        "LESS": "Less",
        "LEX": "Lex",
        "Lua": "lua",
        "Macromedia eXtensible Markup Language": "MXML",
        "Mathematica": "Wolfram",
        "Moonbit": "MoonBit",
        "Patch": "patch",
        "PSL Assertion": "PSL",
        "Powershell": "PowerShell",
        "Protocol Buffers": "Protobuf",
        "ReStructuredText": "reStructuredText",
        "Report Definition Language": "RDL",
        "Rich Text Format": "RTF",
        "Ruby HTML": "ERB",
        "Smarty Template": "Smarty",
        "SNOBOL": "SNOBOL4",
        "SRecode Template": "SRecode",
        "Scons": "SCons",
        "Shell": "sh",
        "Spice Netlist": "SPICE",
        "Standard ML (SML)": "SML",
        "Swig": "SWIG",
        "TCL": "Tcl",
        "Twig Template": "Twig",
        "Varnish Configuration": "VCL",
        "Vim Script": "Vim script",
        "Visual Basic for Applications": "VBA",
        "W.I.S.E. Jobfile": "WISE Jobfile",
        "Web Services Description Language": "WSDL",
        "WebGPU Shading Language": "WGSL",
        "WebGPU Enhanced Shading Language": "WESL",
        "Zsh": "zsh",
        "bait": "Bait",
        "hoon": "Hoon",
        "nuspec": "NuSpec",
        "wenyan": "Wenyan",
        # These are questionable
        "Assembly": "asm",
        "BASH": "bash",
        "CloudFormation (YAML)": "CloudFormation",
        "Device Tree": "DeviceTree",
        "Docker ignore": "DockerIgnore",
        "Emacs Dev Env": "EmacsDevEnv",
        "Extensible Stylesheet Language Transformations": "XSLT",
        "FORTRAN Legacy": "Fortran77",
        "Fortran Modern": "Fortran90",
        "Go Template": "GoTemplate",
        "IDL": "WebIDL",  # IDL is badly ambiguous
        "Plain Text": "text",
        "Windows Resource-Definition Script": "Windows Resource Script",
        "XML Schema": "XSD",
        "gitignore": "ignore",  # *
    }

    def __init__(self):
        with open("tests/scc.json", "r", encoding="utf-8") as rfp:
            self._data = json.load(rfp)
            self._names = set()
            for k in self._data:
                aka = Scc.to_loccount.get(k, k)
                self._data[k]["loccount"] = aka
                self._names.add(aka)

    def name(self):
        return self.__class__.__name__

    def records(self) -> Tuple[Dict]:
        "List of records, with an added 'loccount' member that's the loccount name."
        return tuple({"Name": k, **v} for k, v in self._data.items())

    def loccount_set(self) -> Set[str]:
        return self._names


class Gocloc:
    path = "tests/language.go"
    extractor_pattern = r'"([^"]*)":\s+"([^"]*)",'
    start_with = "var Exts"
    end_with = "}"
    expect = 180
    to_loccount: Final[dict[str, str]] = {
        "XML resource": ".NET Resource",
        "XML Resource": ".NET Resource",
        **common_synonyms,
    }

    def __init__(self):
        state = 0
        self._literal_names = set()
        langdict = {}
        extractor = re.compile(Gocloc.extractor_pattern)
        with open(Gocloc.path, "r", encoding="utf-8") as rfp:
            for line in rfp:
                line = line.strip()
                match state:
                    case 0:
                        if Gocloc.start_with in line:
                            state = 1
                    case 1:
                        if Gocloc.end_with in line:
                            break
                        m = extractor.search(line)
                        if m:
                            ext = m.group(1)
                            name = m.group(2)
                            self._literal_names.add(name)
                            if name not in Cloc.ignore:
                                if name not in langdict:
                                    langdict[name] = []
                                langdict[name].append(ext)
        self._data = [
            {
                "Name": k,
                "loccount_name": Gocloc.to_loccount.get(k, k),
                "Ext": langdict[k],
            }
            for k in self._literal_names
        ]
        self._names = {v["loccount_name"] for v in self._data}
        if len(self._data) < Gocloc.expect:
            print(f"Warning: {Gocloc.path} may be corrupted!")

    def name(self):
        return self.__class__.__name__

    def records(self) -> Tuple[Dict]:
        "Record list is dicts native-names/loccount_name/extension-list elemebts."
        return tuple(self._data)

    def loccount_set(self) -> Set[str]:
        return self._names


class Cloc:
    path = "tests/cloc"
    start_list = "ABAP"
    end_list = ");"
    extractor_pattern = r"\s*'([^']+)'\s*=>\s*'([^']+)'"
    expect = 360
    to_loccount = {
        "Activiti Business Process": "BPMN",
        "ANTLR Grammar": "ANTLR",
        "Apex Trigger": "Apex",  # *
        "ArkTs": "ArkTS",
        "Bourne Again Shell": "bash",
        "Cake Build Script": "Cake",
        "CCS": "CCScript",
        "CoCoA 5": "CoCoA",
        "Constraint Grammar": "ConstraintGrammar",
        "Containerfile": "Dockerfile",
        "D/dtrace": "DTrace",
        "DAL": "DAML",  # Appears to be a typo in Cloc
        "DOORS Extension Language": "DOORS DXL",
        "DOS Batch": "batchfile",
        "Elixir Script": "Elixir",
        "Finite State Language": "FSL",
        "Fish Shell": "fish",
        "Flatbuffers": "FlatBuffers",
        "Fortran 77": "Fortran77",  # *
        "Fortran 2003": "Fortran90",  # *
        "Fortran 90": "Fortran90",  # *
        "Fortran 95": "Fortran90",  # *
        "Freemarker Template": "FTL",
        "Gencat NLS": "GenCat NLS",
        "Glade": "Glade XML",
        "Glimmer JavaScript": "GlimmerJS",
        "Glimmer TypeScript": "GlimmerTS",
        "Godot Shaders": "GDShader",
        "IDL": "WebIDL",
        "IPL": "IPL-V",
        "Javascript": "JavaScript",
        "JavaServer Faces": "JSF",
        "Jinja Template": "Jinja",
        "Justfile": "Just",
        "Kermit": "Kermit Script",
        "Korn Shell": "ksh",
        "Linker Script": "LinkerScript",
        "Mathematica": "Wolfram",
        "Oracle PL/SQL": "PL/SQL",
        "Org Mode": "Org",
        "PL/I": "PL/1",
        "Prisma Schema": "Prisma",
        "Properties": "Properties File",
        "PO File": "PO",
        "Rmd": "RMarkdown",
        "RobotFramework": "Robot Framework",
        "Teamcenter met": "Teamcenter MET",
        "Teamcenter mth": "Teamcenter MTH",
        "Text": "text",
        "TTCN": "TTCN-3",
        "Velocity Template Language": "Velocity",
        "Visual Basic .NET": "Visual Basic",
        "Visual Fox Pro": "xBase",
        "Vuejs Component": "Vue",
        "Windows Message File": "Windows Message",
        "Windows Resource File": "Windows Resource Script",
        "WiX include": "WiX",
        "WiX source": "WiX",
        "WiX string localization": "WiX",
        "Web Services Description": "WSDL",
        "awk": "AWK",
        "builder": "Eclipse Builder Configuration",
        "dhall": "Dhall",
        "liquid": "Liquid",
        "make": "Makefile",
        "kvlang": "KVLang",
        "peg.js": "PEG.js",
        "peggy": "Peggy",
        "tspeg": "tsPEG",
        "vim script": "Vim script",
        "xBase Header": "xBase",
        "yacc": "Yacc",
        **common_synonyms,
    }
    # Yuck. This program can inly detect by file extension,
    # It reposts collisions in anhelpful way. Ignore these,
    ignore = {
        "ADSO/IDSM",
        "C#/Smalltalk",
        "C/C++ Header",
        "Clojure/Cangjie",
        "F#/Forth",
        "Fortran 77/Forth",
        "IDL/Qt Project/Prolog/ProGuard",
        "Jasper Report XML/Template",
        "Lisp/Julia",
        "Lisp/OpenCL",
        "MATLAB/Mathematica/Objective-C/MUMPS/Mercury",
        "Pascal/Pawn",
        "Pascal/Puppet",
        "PHP/Pascal/Fortran/Pawn/BitBake",
        "Perl/Prolog",
        "Raku/Prolog",
        "Scheme/SaltStack",
        "SKILL/.NET IL",
        "SQL Data",  # See the designer's notes
        "SQL Stored Procedure",  # Cloc ovrreached itsef here
        "Unknown/BitBake",
        "TypeScript/Qt Linguist",
        "Vala Header",  # We add .vapi to the Vala extensuins
        "Verilog-SystemVerilog",
        "Verilog-SystemVerilog/Coq",
        "Visual Basic/TeX/Apex Class",
        "XML-Qt-GTK/Glade",
    }

    def __init__(self):
        langdict = {}
        state = 0
        field_extractor = re.compile(Cloc.extractor_pattern)
        expect = 380
        with open(Cloc.path, "r", encoding="utf-8") as rfp:
            for line in rfp:
                line = line.strip()
                match state:
                    case 0:
                        if Cloc.start_list in line:
                            state = 1
                    case 1:
                        if Cloc.end_list in line:
                            break
                        m = field_extractor.search(line)
                        if m:
                            ext = m.group(1)
                            name = m.group(2)
                            if name not in Cloc.ignore:
                                if name not in langdict:
                                    langdict[name] = []
                                langdict[name].append(ext)
        self._data = [
            {
                "Name": name,
                "loccount_name": Cloc.to_loccount.get(name, name),
                "Ext": extensions,
            }
            for name, extensions in langdict.items()
        ]
        if len(self._data) < Cloc.expect:
            print(f"Warning: {Cloc.path} may be corrupted!")

    def name(self):
        return self.__class__.__name__

    def records(self) -> Tuple[Dict]:
        "Record list is just the list of native-names/loccount_name pairs."
        return self._data

    def loccount_set(self) -> Set[str]:
        return {v["loccount_name"] for v in self._data}


def all_foreign():
    return (Gocloc(), Tokei(), Scc(), Cloc())


def asciidoc_count_table(counters) -> None:
    print('[options="header"]')
    print("|=======================")
    print("| Languages | Counters  ")
    stats = [(len(counter.loccount_set()), counter.name()) for counter in counters]
    stats.sort(reverse=True)
    for count, name in stats:
        print(f"| {count:<9} | {name:11}")
    print("|=======================")


def pprint_nonempty(d):
    """
    Return a pretty-printed string of a copy of dictionary d after removing
    entries whose value is "" or [] or our special 'nil'.
    """
    cleaned = {
        k: v
        for k, v in d.items()
        if not (v == "" or v == "nil" or (isinstance(v, list) and len(v) == 0))
    }
    return pformat(cleaned)


def wclpatch(_name, _description, _variant, lang: dict):
    if "Fl" in lang:
        flags = split_mask_flags(lang["Fl"])
        if "asm" in flags:
            lang["Bc"] = [[leader, "\n"] for leader in [";", "*", "//", "#"]]
            flags = [flag for flag in flags if flag != "asm"]
        if "hc" in flags:
            flags = [flag for flag in flags if flag != "hc"]
            if "Bc" in lang:
                lang["Bc"].append(["#", "\n"])
            else:
                lang["Bc"] = [["#", "\n"]]
        lang["Fl"] = join_mask_flags(flags)
        # if not lang["Fl"]:
        #    del lang["F1"]


def experimental():
    with open("mutated.yaml", "wb") as wfp:
        surgeon("languages.yaml", wclpatch, wfp)


if __name__ == "__main__":
    try:
        parsed_args: tuple[list[tuple[str, str]], list[str]] = getopt.getopt(
            sys.argv[1:], "c:g:l:sTtvx"
        )
        options, arguments = parsed_args
    except getopt.GetoptError as e:
        print(e)
        sys.exit(1)
    compare = None
    target = None
    foreign = False
    stats = False
    count_table = False
    quirk = None

    for switch, val in options:
        if switch == "-c":
            compare = val
        elif switch == "-g":
            target = val
        elif switch == "-s":
            stats = True
        elif switch == "-T":
            count_table = True
        elif switch == "-t":
            foreign = True
        elif switch == "-l":
            quirk = val
        elif switch == "-v":
            verbose = True
        elif switch == "-x":
            experimental()
            raise SystemExit()

    # Fake up a class with the interface of Scc/Tokei/Gocloc
    homebase = Loccount()

    for lang in kill_list:
        if lang in homebase.loccount_set():
            print(f"{lang} is both described and on the kill list", file=sys.stderr)

    if target:
        with open(target, "wb") as wfp:
            homebase.go_dump(wfp)
    elif count_table:
        asciidoc_count_table((homebase,) + all_foreign())
    elif foreign:
        print(f"{len(kill_list)} items on the kill list.")
        allrefused: list[tuple[str, set[str]]] = []
        for foreign in (homebase,) + all_foreign():
            total = len(foreign.loccount_set())
            refused = foreign.loccount_set() & kill_list
            allrefused.append((foreign.name(), refused))
            refused = list(refused)
            refused.sort()
            residuals = list(
                foreign.loccount_set() - homebase.loccount_set() - kill_list
            )
            residuals.sort()
            print(f"{foreign.name()} ({total}):")
            if verbose:
                print(f"    Refused: {len(refused)} {refused}")
            print(f"    Residual: {len(residuals)} {residuals}")
        if verbose:
            repeated: dict[str, list[str]] = {}
            for source, refused in allrefused:
                for entry in refused:
                    repeated.setdefault(entry, []).append(source)
            repeated = {
                entry: sources
                for entry, sources in repeated.items()
                if len(sources) > 1
            }
            if repeated:
                print("Repeated refused entries:")
                for entry, sources in sorted(
                    repeated.items(), key=lambda item: (-len(item[1]), item[0])
                ):
                    print(
                        f"    {entry}: {len(sources)} " f"lists ({', '.join(sources)})"
                    )
            else:
                print("Repeated refused entries: none")
    elif quirk:
        lst = list(
            entry["Name"]
            for entry in homebase.records()
            if quirk in split_mask_flags(entry.get("Fl", ""))
        )
        print(f"({len(lst)}) {lst}")
    elif stats:
        total = 0
        dsls = 0
        tags = tuple(classes.items()) + (
            ("xprop", "proprietary"),
            ("xdead", "dead"),
        )
        keys = {tagpair[0] for tagpair in tags}
        dslkinds = keys.copy()
        dslkinds.remove("xdead")
        dslkinds.remove("xprop")
        dslkinds.remove("xdsl")
        counts = {k: 0 for k in keys}
        records = homebase.records()
        for n, entry in enumerate(records):
            if "Ext" in entry or "Files" in entry:
                if n > 0 and records[n]["Name"] == records[n - 1]["Name"]:
                    continue
                total += 1
                is_dsl = False
                for tag in split_mask_flags(entry.get("Fl", "")):
                    if tag in keys:
                        counts[tag] += 1
                        if tag in dslkinds:
                            is_dsl = True
                if is_dsl:
                    dsls += 1
        print("Accepted:")
        print(f"{total:3} total entries")
        print(f"{total - dsls:3} programming languages")
        print(f"{dsls:3} DSLs, all kinds")
        for tag, legend in tags:
            print(f"{counts[tag]:3} {legend}")
        print(f"Rejected ({len(kill_list)}):")
        print(f"{len(proprietary_only):3} tied proprietary")
        print(f"{len(toy_languages):3} toys")
        print(f"{len(exceptions_and_mixed):3} exceptions")
    elif compare:
        m = re.compile(compare, re.IGNORECASE)
        for counter in (homebase,) + all_foreign():
            print(f"{counter.name()}:")
            latch = False
            for entry in counter.records():
                textual = str(entry)
                if m.search(textual):
                    print("    " + pprint_nonempty(entry).replace("\n", "\n    "))
                    latch = True
            if not latch:
                print(f"    No match in {counter.name()}")

# end
