// SPDX-License-Identifier: BSD-2-Clause
package main

//go:generate ./tablegen.py -g traits.go

import (
	"bufio"
	"bytes"
	"encoding/xml"
	"flag"
	"fmt"
	"github.com/boyter/gocodewalker"
	"go/ast"
	"go/parser"
	"go/token"
	"html"
	"io"
	"log"
	"math"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"runtime/pprof"
	"sort"
	"strings"
	"sync"
)

var version string

// const cSALARY = 79000 // From Wikipedia, late 2018
const cSALARY = 130000 // From Grok, mid-2026

// SourceStat - line count record for a specified path
type SourceStat struct {
	Path         string
	Language     string
	SLOC         uint
	LLOC         uint
	contentMatch bool
	markup       bool
	cgi          bool
}

func displayLanguage(language string, cgi bool) string {
	if language == "" {
		return ""
	}
	if cgi {
		return language + " CGI"
	}
	return language
}

func (s SourceStat) reportLanguage() string {
	return displayLanguage(s.Language, s.cgi)
}

var debug int
var transdump bool
var listignores bool
var exclusions *regexp.Regexp
var pipeline chan SourceStat
var stringMatchMu sync.Mutex
var stringMatchLanguages = make(map[string]map[string]struct{})
var extensionLanguageCounts map[string]int
var extensionLanguageCountsOnce sync.Once

func (g genericLanguage) property(v uint) bool {
	return (v & g.Fl) != 0
}

// fileMatches() tests whether a file matches a known special fiile name
// of this language
func (g genericLanguage) fileMatches(path string) string {
	base := filepath.Base(path)
	for _, s := range g.Files {
		if base == s {
			return s
		}
	}
	return ""
}

// patchMatches() tests whether a file matches this language by name or extension
func (g genericLanguage) pathMatches(path string, ext string) string {
	for _, s := range g.Ext {
		if s == ext && strings.HasSuffix(path, s) {
			return s
		}
	}
	return g.fileMatches(path)
}

// matchedExtensionSuffix() tells whether a string has a known extension
func matchedExtensionSuffix(path string) string {
	lopath := strings.ToLower(path)
	best := ""
	for i := range genericLanguages {
		for _, ext := range genericLanguages[i].Ext {
			if strings.HasSuffix(lopath, ext) && len(ext) > len(best) {
				best = ext
			}
		}
	}
	return best
}

func trackStringMatch(path string, language string) {
	ext := matchedExtensionSuffix(path)
	if ext == "" {
		return
	}
	stringMatchMu.Lock()
	defer stringMatchMu.Unlock()
	languages := stringMatchLanguages[ext]
	if languages == nil {
		languages = make(map[string]struct{})
		stringMatchLanguages[ext] = languages
	}
	languages[language] = struct{}{}
}

func uniqueStringMatchAssignments() map[string]string {
	assignments := make(map[string]string)
	for ext, languages := range stringMatchLanguages {
		if len(languages) != 1 {
			continue
		}
		for language := range languages {
			assignments[ext] = language
		}
	}
	return assignments
}

func uniqueStringMatchExtensions() []string {
	assignments := uniqueStringMatchAssignments()
	extensions := make([]string, 0, len(assignments))
	for ext := range assignments {
		extensions = append(extensions, ext)
	}
	sort.Strings(extensions)
	return extensions
}

func initExtensionLanguageCounts() {
	counts := make(map[string]map[string]struct{})
	for i := range genericLanguages {
		langName := genericLanguages[i].Name
		for _, ext := range genericLanguages[i].Ext {
			languages := counts[ext]
			if languages == nil {
				languages = make(map[string]struct{})
				counts[ext] = languages
			}
			languages[langName] = struct{}{}
		}
	}
	flattened := make(map[string]int, len(counts))
	for ext, languages := range counts {
		flattened[ext] = len(languages)
	}
	extensionLanguageCounts = flattened
}

func sharedLanguageExtension(ext string) bool {
	extensionLanguageCountsOnce.Do(initExtensionLanguageCounts)
	return extensionLanguageCounts[ext] > 1
}

func syntaxForLanguageExtension(language string, ext string) (genericLanguage, bool) {
	for i := range genericLanguages {
		lang := genericLanguages[i]
		if lang.Name != language {
			continue
		}
		for _, candidate := range lang.Ext {
			if candidate == ext {
				return lang, true
			}
		}
	}
	for i := range genericLanguages {
		lang := genericLanguages[i]
		if lang.Name == language {
			return lang, true
		}
	}
	return genericLanguage{}, false
}

var neverInterestingByPrefix []string
var actuallyInterestingByPrefix []string
var neverInterestingByInfix []string
var neverInterestingBySuffix map[string]bool
var neverInterestingByBasename map[string]bool
var blankLineRE = regexp.MustCompile(`\n\s*\n`)
var yamlBlockScalarRE = regexp.MustCompile(`^\s*(?:(?:---|\.\.\.)\s+)?(?:-\s+|\?\s+)?(?:[!&]\S+\s+)*(?:[|>](?:[1-9][+-]?|[+-][1-9]?|)\s*(?:#.*)?|.*:\s*(?:[!&]\S+\s+)*[|>](?:[1-9][+-]?|[+-][1-9]?|)\s*(?:#.*)?)$`)

var cHeaderPriority []string
var generated string

func init() {
	neverInterestingByPrefix = []string{"."}
	actuallyInterestingByPrefix = []string{"../", "./"}
	neverInterestingByInfix = []string{".so.", "/."}
	// This is a fast path; it's OK if this is leaky
	// because binaries are highly likely to be caught
	// by the check for early NUL. Matching here means
	// we get away with not having to open the file.
	ignoreExt := []string{"~",
		".a", ".la", ".o", ".so", ".ko",
		".gif", ".jpg", ".jpeg", ".ico", ".xpm", ".xbm",
		".bmp", ".webp", ".png", ".tiff", ".psd",
		".pdf", ".eps",
		".tfm", ".ttf", ".bdf", ".afm",
		".fig", ".pic",
		".pyc", ".pyo", ".elc", ".wasm",
		".n", ".po",
		".gz", ".bz2", ".Z", ".tgz", ".zip",
		".au", ".wav", ".ogg",
		".exe", ".dll", ".obj", ".class", ".jar",
		".mp3", ".mp4", ".mkv",
		".xz", ".zst", ".7z",
		".db", ".sqlite3",
		".docx", ".xlsx", ".pptx",
		".otf", ".woff2",
		".min.js", ".map",
	}
	neverInterestingBySuffix = make(map[string]bool)
	for i := range ignoreExt {
		neverInterestingBySuffix[ignoreExt[i]] = true
	}
	neverInterestingByBasename = map[string]bool{
		// FIXME: Should we exclude READMEs now that we're counting docs?
		"readme": true, "readme.tk": true, "readme.md": true, "readme.adoc": true,
		"changelog": true, "repository": true, "changes": true,
		"bugs": true, "todo": true, "copying": true, "maintainers": true,
		"news":      true,
		"configure": true, "autom4te.cache": true, "config.log": true,
		"config.status": true,
		"lex.yy.c":      true, "lex.yy.cc": true,
		"y.code.c": true, "y.tab.c": true, "y.tab.h": true,
		"asciidoc.js": true,
		"sccs":        true, "rcs": true,
		"waf": true, "package-lock.json": true,
		".designer.cs": true,
	}
	cHeaderPriority = []string{"C", "C++", "Objective C"}

	generated = "GENERATED|automatically generated|generated automatically|generated by|a lexical scanner generated by flex|this is a generated file|generated with the.*utility|do not edit|do not hand-hack|DocBook XSL Stylesheets|<meta name=\"generator\" content=\"AsciiDoc|[!]PS-Adobe"

}

// Generic machinery for walking source text to count lines

type delimSpec struct {
	Open  string
	Close string
	Flags uint
}

type activeDelim struct {
	spec          delimSpec
	end           string
	outdentIndent uint
}

type openerRegexp struct {
	re     *regexp.Regexp
	prefix string
}

var openerRegexps map[string]openerRegexp
var openerRegexpsOnce sync.Once

// countContext is state corresponding to a single source file
type countContext struct {
	line             []byte
	lineNumber       uint        // Line number in file
	charCount        uint        // Character index on line
	charConsumed     uint        // Characters consumed on line
	lineIndent       uint        // Leading spaces consumed on current line
	lexsection       uint        // Count %% sections in a lex file
	wasPreRegexp     bool        // In the plausible set for a regexp leader?
	commentStack     []delimSpec // How many levels deep in block comments are we?
	startline        uint
	startchar        uint
	quoteDelims      []activeDelim
	newlineLatch     bool
	lastChar         byte
	lastToken        string // Await this end-of-literal token
	slocEligible     bool   // If true this line can be counted as SLOC
	lineRegion       bool
	heredoc          string
	underlyingStream *os.File
	rc               *bufio.Reader
}

func (ctx *countContext) setup(path string) bool {
	var err error
	ctx.underlyingStream, err = os.Open(path)
	if err != nil {
		log.Println(err)
		return false
	}
	ctx.rc = bufio.NewReader(ctx.underlyingStream)
	ctx.lineNumber = 1
	ctx.charCount = 0
	ctx.charConsumed = 0
	ctx.lineIndent = 0
	ctx.slocEligible = false
	ctx.wasPreRegexp = false
	ctx.newlineLatch = false
	ctx.lineRegion = false
	ctx.heredoc = ""
	if debug > 2 {
		ctx.breadcrumb("All counters zeroed")
	}
	return true
}

// teadown() de-initializes the I/O stream for a count
func (ctx *countContext) teardown() {
	ctx.underlyingStream.Close()
}

// countContext) saves a start location for use in a later breadcrumb call
func (ctx *countContext) saveMark() {
	ctx.startline = ctx.lineNumber
	ctx.startchar = ctx.charConsumed
}

// wasNewline() says wheter the last character consumed was a newline
func (ctx *countContext) wasNewline() bool {
	return ctx.charCount == 0 || ctx.charConsumed == 0
}

// breadcrumb() reports an error location in the style of a GCC error message
func (ctx *countContext) breadcrumb(msg string) {
	var ln uint
	var off uint
	ln = ctx.lineNumber
	off = ctx.charConsumed + 1
	fmt.Fprintf(os.Stderr, "%s:%d:%d: %s\n", ctx.underlyingStream.Name(), ln, off, msg)
}

// markEligible() marks current line elegible to be counted as a logical line
func (ctx *countContext) markEligible() {
	if !ctx.slocEligible {
		ctx.slocEligible = true
		if debug > 1 {
			ctx.breadcrumb("marked eligible")
		}
	}
}

// updateCounters updates all stat counters and location trackers for current char
func (ctx *countContext) updateCounters(c byte) {
	if debug > 3 {
		ctx.breadcrumb(fmt.Sprintf("updating counters after %q", []byte{c}))
	}
	ctx.charCount++
	if c == '\n' {
		ctx.lineNumber++
		ctx.charConsumed = 0
		ctx.lineIndent = 0
		ctx.slocEligible = false
	} else {
		ctx.charConsumed++
		if ctx.charConsumed == ctx.lineIndent+1 && c == ' ' {
			ctx.lineIndent++
		}
	}
	ctx.lastChar = c
}

// expectConst consumes a string returning true, or returns false on no match
func (ctx *countContext) expectConst(exp string) bool {
	if len(exp) == 0 {
		if debug > 3 {
			ctx.breadcrumb(fmt.Sprintf("expectConst %q returns false.", exp))
		}
		return false
	}
	var s []byte
	var err error
	var line int
	var fn string
	if debug > 2 {
		_, fn, line, _ = runtime.Caller(1)
	}
	if s, err = ctx.rc.Peek(len(exp)); err == nil && string(s) == exp {
		ctx.rc.Discard(len(exp))
		if debug > 2 {
			ctx.breadcrumb(fmt.Sprintf("expectConst %q at %s:%d returns true", exp, fn, line))
		} else if transdump {
			os.Stdout.Write(s)
		}
		for i := range s {
			ctx.updateCounters(s[i])
		}
		ctx.lastToken = string(s)
		return true
	}
	if debug > 3 {
		ctx.breadcrumb(fmt.Sprintf("expectConst %q at %s:%d saw %q returns false", exp, fn, line, s))
	}
	return false
}

// expectRegexp consumes a regexp match, or returns false on no match
func (ctx *countContext) expectRegexp(exp *regexp.Regexp) (bool, string) {
	if exp == nil {
		if debug > 3 {
			ctx.breadcrumb("expectRegexp <nil> returns false.")
		}
		return false, ""
	}

	var line int
	var fn string
	if debug > 2 {
		_, fn, line, _ = runtime.Caller(1)
	}

	offset, err := ctx.underlyingStream.Seek(0, io.SeekCurrent)
	if err != nil {
		if debug > 3 {
			ctx.breadcrumb(fmt.Sprintf("expectRegexp %q at %s:%d seek failed: %v", exp.String(), fn, line, err))
		}
		return false, ""
	}
	offset -= int64(ctx.rc.Buffered())

	buf := make([]byte, 0, 128)
	scratch := make([]byte, 128)
	for {
		n, readErr := ctx.underlyingStream.ReadAt(scratch, offset+int64(len(buf)))
		if n > 0 {
			buf = append(buf, scratch[:n]...)
			if idx := bytes.IndexByte(buf, '\n'); idx >= 0 {
				buf = buf[:idx+1]
				break
			}
		}
		if readErr == io.EOF {
			break
		}
		if readErr != nil {
			if debug > 3 {
				ctx.breadcrumb(fmt.Sprintf("expectRegexp %q at %s:%d read failed: %v", exp.String(), fn, line, readErr))
			}
			return false, ""
		}
	}

	loc := exp.FindSubmatchIndex(buf)
	if loc == nil || loc[0] != 0 {
		if debug > 3 {
			ctx.breadcrumb(fmt.Sprintf("expectRegexp %q at %s:%d saw %q returns false", exp.String(), fn, line, buf))
		}
		return false, ""
	}

	matched := buf[:loc[1]]
	ctx.rc.Discard(len(matched))
	if debug > 2 {
		ctx.breadcrumb(fmt.Sprintf("expectRegexp %q at %s:%d returns true", exp.String(), fn, line))
	} else if transdump {
		os.Stdout.Write(matched)
	}
	for i := range matched {
		ctx.updateCounters(matched[i])
	}
	ctx.lastToken = string(matched)

	group := ""
	if exp.NumSubexp() > 0 && len(loc) >= 4 && loc[2] >= 0 && loc[3] >= 0 {
		group = string(buf[loc[2]:loc[3]])
	}
	return true, group
}

// hasPrefix checks for a specified string prefix without consuming
func (ctx *countContext) hasPrefix(exp string) bool {
	if len(exp) == 0 {
		return false
	}
	s, err := ctx.rc.Peek(len(exp))
	return err == nil && string(s) == exp
}

func (ctx *countContext) hasLineCloser(exp string) bool {
	if len(exp) == 0 {
		return false
	}
	s, err := ctx.rc.Peek(len(exp) + 1)
	if len(s) < len(exp) || string(s[:len(exp)]) != exp {
		return false
	}
	if len(s) == len(exp) {
		return err == io.EOF
	}
	return s[len(exp)] == '\n'
}

func (ctx *countContext) expectIndentedLineCloser(exp string) bool {
	if len(exp) == 0 {
		return false
	}
	line, ok := ctx.peekLine()
	if !ok {
		return false
	}
	indent := 0
	for indent < len(line) && (line[indent] == ' ' || line[indent] == '\t') {
		indent++
	}
	rest := line[indent:]
	if len(rest) < len(exp) || string(rest[:len(exp)]) != exp {
		return false
	}
	if len(rest) > len(exp) && rest[len(exp)] != '\n' {
		return false
	}
	matchedLen := indent + len(exp)
	matched, err := ctx.rc.Peek(matchedLen)
	if err != nil || len(matched) != matchedLen {
		return false
	}
	ctx.rc.Discard(matchedLen)
	if transdump {
		os.Stdout.Write(matched)
	}
	for i := range matched {
		ctx.updateCounters(matched[i])
	}
	ctx.lastToken = string(matched)
	return true
}

func hasPairFlag(part delimSpec, flag uint) bool {
	return part.Flags&flag != 0
}

func compiledOpenerRegexp(pattern string) openerRegexp {
	openerRegexpsOnce.Do(func() {
		openerRegexps = make(map[string]openerRegexp)
		for _, language := range genericLanguages {
			for _, pair := range language.Bc {
				if !hasPairFlag(pair, re) {
					continue
				}
				re, err := regexp.Compile(pair.Open)
				if err != nil {
					panic(fmt.Sprintf("unexpected failure while compiling %s", pair.Open))
				}
				if re.NumSubexp() > 1 {
					panic(fmt.Sprintf("regexp opener %q has more than one capture group", pair.Open))
				}
				if re.NumSubexp() == 0 && strings.Contains(pair.Close, "&") {
					panic(fmt.Sprintf("regexp opener %q uses '&' in end template without a capture group", pair.Open))
				}
				prefix, _ := re.LiteralPrefix()
				openerRegexps[pair.Open] = openerRegexp{re: re, prefix: prefix}
			}
			for _, pair := range language.Str {
				if !hasPairFlag(pair, re) {
					continue
				}
				re, err := regexp.Compile(pair.Open)
				if err != nil {
					panic(fmt.Sprintf("unexpected failure while compiling %s", pair.Open))
				}
				if re.NumSubexp() > 1 {
					panic(fmt.Sprintf("regexp opener %q has more than one capture group", pair.Open))
				}
				if re.NumSubexp() == 0 && strings.Contains(pair.Close, "&") {
					panic(fmt.Sprintf("regexp opener %q uses '&' in end template without a capture group", pair.Open))
				}
				prefix, _ := re.LiteralPrefix()
				openerRegexps[pair.Open] = openerRegexp{re: re, prefix: prefix}
			}
		}
	})
	opener, ok := openerRegexps[pattern]
	if !ok {
		panic(fmt.Sprintf("missing compiled opener regexp for %q", pattern))
	}
	return opener
}

// resolveDelimiterTemplate() performs & substitution in a regexp closer
func resolveDelimiterTemplate(template string, capture string) string {
	if !strings.Contains(template, "&") {
		return template
	}
	var b strings.Builder
	b.Grow(len(template) + len(capture))
	for i := 0; i < len(template); i++ {
		if template[i] != '&' {
			b.WriteByte(template[i])
			continue
		}
		if i+1 < len(template) && template[i+1] == '&' {
			b.WriteByte('&')
			i++
			continue
		}
		b.WriteString(capture)
	}
	return b.String()
}

func lineCommentLeaders(lang genericLanguage) []string {
	var leaders []string
	for _, pair := range lang.Bc {
		if pair.Close == "\n" && pair.Open != "" {
			leaders = append(leaders, pair.Open)
		}
	}
	return leaders
}

func (ctx *countContext) expect(part delimSpec) (bool, string) {
	if !hasPairFlag(part, re) {
		return ctx.expectConst(part.Open), ""
	}
	opener := compiledOpenerRegexp(part.Open)
	if opener.prefix != "" && !ctx.hasPrefix(opener.prefix) {
		return false, ""
	}
	return ctx.expectRegexp(opener.re)
}

// eof() is a predicate tje tells us if we're at end of file
func (ctx *countContext) eof() bool {
	_, err := ctx.rc.Peek(1)
	return err == io.EOF
}

// munchline() consumes the remainder of a line, updating the line counter
func (ctx *countContext) munchline() bool {
	line, err := ctx.rc.ReadBytes('\n')
	if err == nil {
		ctx.lineNumber++
		ctx.line = line
		ctx.charCount = uint(len(line))
		ctx.charConsumed = ctx.charCount
		return true
	} else if err == io.EOF {
		if len(line) == 0 {
			return false
		}
		// Last line without trailing newline still counts as a line.
		ctx.lineNumber++
		ctx.line = line
		ctx.charCount = uint(len(line))
		ctx.charConsumed = ctx.charCount
		return true
	} else {
		panic(err)
	}
}

// matchline() does a given regexp match the last line read?
func (ctx *countContext) matchline(re string) bool {
	cre, err := regexp.Compile(re)
	if err != nil {
		panic(fmt.Sprintf("unexpected failure %s while compiling %s", err, re))
	}
	return cre.Find(ctx.line) != nil
}

// peekLine() peeks at the bext line in the stream without consuming it
func (ctx *countContext) peekLine() ([]byte, bool) {
	n := 1
	for {
		chunk, err := ctx.rc.Peek(n)
		if err == nil {
			if i := bytes.IndexByte(chunk, '\n'); i >= 0 {
				return chunk[:i+1], true
			}
			n++
			continue
		}
		if err == bufio.ErrBufferFull {
			if i := bytes.IndexByte(chunk, '\n'); i >= 0 {
				return chunk[:i+1], true
			}
			n = len(chunk) + 1
			continue
		}
		if err == io.EOF {
			if len(chunk) == 0 {
				return nil, false
			}
			return chunk, true
		}
		panic(err)
	}
}

func trimLine(line []byte) []byte {
	return bytes.Trim(line, " \t\r\n")
}

// lineIndent() extracts the indentation level of a line
func lineIndent(line []byte) int {
	indent := 0
	for indent < len(line) && (line[indent] == ' ' || line[indent] == '\t') {
		indent++
	}
	return indent
}

// yamlStartsBlockScalar() tells whether a line starts a YANL block literal
func yamlStartsBlockScalar(line []byte) bool {
	return yamlBlockScalarRE.Match(bytes.TrimRight(line, "\r\n"))
}

// detectHeredoc() detects an oncoming heredoc and returns its end delimiter
func detectHeredoc(line []byte) string {
	i := bytes.Index(line, []byte("<<"))
	if i < 0 {
		return ""
	}
	text := bytes.TrimLeft(line[i+2:], " \t")
	if len(text) == 0 {
		return ""
	}
	if text[0] == '\'' || text[0] == '"' || text[0] == '`' {
		quote := text[0]
		end := bytes.IndexByte(text[1:], quote)
		if end < 0 {
			return ""
		}
		return string(text[1 : end+1])
	}
	end := bytes.IndexAny(text, " \t\r\n;,")
	if end < 0 {
		end = len(text)
	}
	return string(text[:end])
}

func isspace(c byte) bool {
	return c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f'
}

func isdigit(c byte) bool {
	return c >= '0' && c <= '9'
}

func isAlphameric(c byte) bool {
	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')
}

func isRegexpWordByte(c byte) bool {
	return isAlphameric(c) || c == '_' || c == '$'
}

func ispunct(c byte) bool {
	return c != 0 && !isAlphameric(c) && !isspace(c)
}

// Verifier functions for checking that files with disputed extensions
// are actually of the types we think they are.

func passesTextChecks(ctx *countContext, path string, lang string, requires []string, forbids []string) bool {
	if len(requires) == 0 && len(forbids) == 0 {
		return true
	}

	if !ctx.setup(path) {
		return false
	}
	defer ctx.teardown()

	requiredMatched := make([]bool, len(requires))
	requiredRemaining := len(requires)

	for ctx.munchline() {
		for i := range forbids {
			if ctx.matchline(forbids[i]) {
				if debug > 0 {
					fmt.Fprintf(os.Stderr, "%s forbids returned true on %s\n",
						lang, path)
					if debug > 1 {
						fmt.Fprintf(os.Stderr, "%s forbids matched %q on %s\n",
							lang, forbids[i], path)
					}
				}
				return false
			}
		}
		for i := range requires {
			if !requiredMatched[i] && ctx.matchline(requires[i]) {
				requiredMatched[i] = true
				requiredRemaining--
			}
		}
	}

	allRequiredMatched := requiredRemaining == 0

	if debug > 0 {
		fmt.Fprintf(os.Stderr, "%s requires returned %t on %s\n",
			lang, allRequiredMatched, path)
		fmt.Fprintf(os.Stderr, "%s forbids returned false on %s\n",
			lang, path)
		if debug > 1 && !allRequiredMatched {
			for i := range requires {
				if !requiredMatched[i] {
					fmt.Fprintf(os.Stderr, "%s requires missing %q on %s\n",
						lang, requires[i], path)
				}
			}
		}
	}

	return allRequiredMatched
}

func wasGeneratedAutomatically(ctx *countContext, path string, commentleaders []string) bool {
	// Determine if the file was generated automatically.
	// Use a simple heuristic: check if first few lines have phrases like
	// "generated automatically", "automatically generated", "Generated by",
	// or "do not edit" as the first
	// words in the line (after possible comment markers and spaces).
	i := 15 // Look at first 15 lines.
	if !ctx.setup(path) {
		return false
	}
	defer ctx.teardown()

	leaders := []string{""}
	for _, leader := range commentleaders {
		if leader != "" {
			leaders = append(leaders, regexp.QuoteMeta(leader))
		}
	}
	re := "(?:" + strings.Join(leaders, "|") + ").*(?i:" + generated + ")"
	if debug > 1 {
		ctx.breadcrumb(fmt.Sprintf("<ignore:%s>", re))
	}
	cre, err := regexp.Compile(re)
	if err != nil {
		panic(fmt.Sprintf("unexpected failure while building %s", re))
	}

	for ctx.munchline() && i > 0 {
		//fmt.Fprintf(os.Stderr, "Matching %s against %s", ctx.line, re)
		if cre.Find(ctx.line) != nil {
			if debug > 0 {
				fmt.Fprintf(os.Stderr, "%s: is generated\n", path)
			}
			return true
		}
		i--
	}

	return false
}

func shebangInterpreter(line string) string {
	s := strings.TrimSpace(line)
	if !strings.HasPrefix(s, "#!") {
		return ""
	}
	fields := strings.Fields(strings.TrimSpace(s[2:]))
	if len(fields) == 0 {
		return ""
	}
	cmd := strings.ToLower(filepath.Base(fields[0]))
	if cmd != "env" {
		return cmd
	}
	for i := 1; i < len(fields); i++ {
		if strings.HasPrefix(fields[i], "-") {
			continue
		}
		return strings.ToLower(filepath.Base(fields[i]))
	}
	return ""
}

// hashbang - match the interpreter name from the first line
func hashbang(ctx *countContext, path string, interpreters []string, allowNonExecutable bool) bool {
	fi, err := os.Stat(path)
	if err != nil {
		return false
	}
	// Normally we only trust hashbangs on executable files, but callers
	// may override that for extensions such as .cgi that imply script use.
	if !allowNonExecutable && (fi.Mode()&01111) == 0 {
		return false
	}
	if !ctx.setup(path) {
		return false
	}
	defer ctx.teardown()
	s, err := ctx.rc.ReadString('\n')
	if err != nil && err != io.EOF {
		return false
	}
	interpreter := shebangInterpreter(s)
	if interpreter == "" {
		return false
	}
	if debug > 0 {
		fmt.Fprintf(os.Stderr, "hashbang checking interpreter %q against alternatives %s\n", interpreter, interpreters)
	}
	for _, alternative := range interpreters {
		if strings.EqualFold(interpreter, alternative) {
			return true
		}
	}
	return false
}

// dumpStransition() dumps a string with an ANSI reverse-video highlight.
// This is used only for debug output.
func dumpTransition(legend string) {
	os.Stdout.WriteString("\x1B[7m" + legend + "\x1B[0m")
}

// genericCounter - Count SLOC (and possibly LLOC) in a source file for a generic procedural language
func genericCounter(ctx *countContext, path string, syntax genericLanguage, ext string) []SourceStat {
	const stateNORMAL = 0   // in running text
	const stateINSTRING = 1 // in single-line string
	const stateCOMMENT = 2  // in any comment
	const stateINREGEXP = 3 // in a regular-expression literal
	var stateNames = [4]string{"NORMAL", "STRING", "COMMENT", "REGEXP"}

	var stats SourceStat
	mode := stateNORMAL

	if !ctx.setup(path) {
		return nil
	}
	defer ctx.teardown()

	stats.Path = path
	stats.Language = syntax.Name
	stats.markup = syntax.property(doc)

	if syntax.property(yaml) {
		inBlockScalar := false
		blockParentIndent := 0

		for {
			line, err := ctx.rc.ReadBytes('\n')
			if err != nil && err != io.EOF {
				panic("error while reading YAML line")
			}
			if len(line) == 0 {
				break
			}

			trimmed := trimLine(line)
			indent := lineIndent(line)

			if inBlockScalar {
				if len(trimmed) == 0 {
					if err == io.EOF {
						break
					}
					continue
				}
				if indent > blockParentIndent {
					stats.SLOC++
					if err == io.EOF {
						break
					}
					continue
				}
				inBlockScalar = false
			}

			if len(trimmed) == 0 || trimmed[0] == '#' {
				if err == io.EOF {
					break
				}
				continue
			}

			stats.SLOC++
			if yamlStartsBlockScalar(line) {
				inBlockScalar = true
				blockParentIndent = indent
			}

			if err == io.EOF {
				break
			}
		}
		return []SourceStat{stats}

	}

	rxTokens := make(map[string]struct{}, len(syntax.Rx))
	for _, token := range syntax.Rx {
		rxTokens[token] = struct{}{}
	}

	rxToken := func(token string) bool {
		_, ok := rxTokens[token]
		return ok
	}

	normalWord := ""
	flushNormalWord := func() {
		if len(rxTokens) == 0 {
			normalWord = ""
			return
		}
		if normalWord == "" {
			return
		}
		ctx.wasPreRegexp = rxToken(normalWord)
		normalWord = ""
	}
	clearNormalWord := func() {
		normalWord = ""
	}

	enterComment := func(pair delimSpec) {
		ctx.commentStack = append(ctx.commentStack, pair)
		if debug > 1 {
			ctx.breadcrumb(fmt.Sprintf("comment level incremented to %d", len(ctx.commentStack)))
		}
	}

	commentStart := func(syntax genericLanguage) bool {
		if syntax.property(f77) || syntax.property(f90) {
			is_directive := func() bool {
				for _, d := range []string{"hpf$", "$omp", "$acc", "dir$"} {
					if ctx.expectConst(d) {
						return true
					}
				}
				return false
			}

			if ctx.expectConst("!") {
				if !is_directive() {
					enterComment(delimSpec{Open: "!", Close: "\n"})
					return true
				}
				return false
			} else if ctx.wasNewline() && syntax.property(f77) {
				if ctx.expectConst("c") || ctx.expectConst("C") || ctx.expectConst("*") || ctx.expectConst("!") {
					if !is_directive() {
						enterComment(delimSpec{Open: ctx.lastToken, Close: "\n", Flags: sol})
						return true
					}
					return false
				}
			}
			return false
		}
		for i, pair := range syntax.Bc {
			atsol := ctx.wasNewline()
			if matched, capture := ctx.expect(pair); matched {
				if hasPairFlag(syntax.Bc[i], sol) && !atsol {
					return false
				}
				flushNormalWord()
				comment := syntax.Bc[i]
				comment.Close = resolveDelimiterTemplate(comment.Close, capture)
				enterComment(comment)
				return true
			}
		}
		return false
	}

	nestedCommentStart := func(syntax genericLanguage) bool {
		for i, pair := range syntax.Bc {
			if pair.Close == "\n" {
				continue
			}
			atsol := ctx.wasNewline()
			if matched, capture := ctx.expect(pair); matched {
				if hasPairFlag(syntax.Bc[i], sol) && !atsol {
					return false
				}
				comment := syntax.Bc[i]
				comment.Close = resolveDelimiterTemplate(comment.Close, capture)
				enterComment(comment)
				return true
			}
		}
		return false
	}

	commentEnd := func() bool {
		if len(ctx.commentStack) == 0 {
			return false
		}
		active := ctx.commentStack[len(ctx.commentStack)-1]
		if active.Close == "\n" {
			return ctx.hasPrefix("\n")
		}
		if hasPairFlag(active, sol) && !ctx.wasNewline() {
			return false
		}
		return ctx.expectConst(active.Close)
	}

	inlineString := func() bool {
		if syntax.property(lit) {
			return false
		}
		for n, qpair := range syntax.Str {
			openerIndent := ctx.lineIndent
			openerEligible := ctx.slocEligible
			if matched, capture := ctx.expect(qpair); matched {
				clearNormalWord()
				ctx.wasPreRegexp = false
				ctx.quoteDelims = append(ctx.quoteDelims, activeDelim{
					spec:          syntax.Str[n],
					end:           resolveDelimiterTemplate(qpair.Close, capture),
					outdentIndent: openerIndent,
				})
				if openerEligible && strings.Contains(ctx.lastToken, "\n") {
					stats.SLOC++
				}
				return true
			}
		}
		return false
	}

	stringStart := func() bool {
		if !syntax.property(tcl) {
			return inlineString()
		}
		if !(ctx.wasNewline() || isspace(ctx.lastChar) || ispunct(ctx.lastChar)) {
			return false
		}
		return inlineString()
	}

	stringEnd := func() bool {
		if len(ctx.quoteDelims) == 0 {
			return false
		}
		active := ctx.quoteDelims[len(ctx.quoteDelims)-1]
		if hasPairFlag(active.spec, outdent) {
			if ctx.charConsumed != ctx.lineIndent {
				return false
			}
			if ctx.lineIndent > active.outdentIndent {
				return false
			}
			next, err := ctx.rc.Peek(1)
			if err != nil || len(next) == 0 {
				return false
			}
			return next[0] != ' ' && next[0] != '\n'
		}
		if hasPairFlag(active.spec, sol) {
			if !ctx.wasNewline() {
				return false
			}
			if hasPairFlag(active.spec, ind) {
				return ctx.expectIndentedLineCloser(active.end)
			}
			if !ctx.hasLineCloser(active.end) {
				return false
			}
		}
		return ctx.expectConst(active.end)
	}

	eol := func() bool {
		return ctx.expectConst("\n")
	}

	regexpCloserAhead := func() bool {
		offset, err := ctx.underlyingStream.Seek(0, io.SeekCurrent)
		if err != nil {
			return false
		}
		offset -= int64(ctx.rc.Buffered())

		const maxRegexpLookahead = 8192
		scratch := make([]byte, 1024)
		scanned := 0
		escaped := false
		inClass := false
		for scanned < maxRegexpLookahead {
			want := len(scratch)
			if remaining := maxRegexpLookahead - scanned; remaining < want {
				want = remaining
			}
			n, readErr := ctx.underlyingStream.ReadAt(scratch[:want], offset+int64(scanned))
			for i := 0; i < n; i++ {
				if scanned == 0 && i == 0 {
					if scratch[i] != '/' {
						return false
					}
					continue
				}
				c := scratch[i]
				if c == '\n' || c == '\r' {
					return false
				}
				if escaped {
					escaped = false
					continue
				}
				if c == '\\' {
					escaped = true
					continue
				}
				if c == '[' && !inClass {
					inClass = true
					continue
				}
				if c == ']' && inClass {
					inClass = false
					continue
				}
				if c == '/' && !inClass {
					return true
				}
			}
			scanned += n
			if readErr == io.EOF {
				return false
			}
			if readErr != nil || n == 0 {
				return false
			}
		}
		return false
	}

	regexpStart := func() bool {
		if len(rxTokens) == 0 || !ctx.hasPrefix("/") {
			return false
		}
		flushNormalWord()
		if !ctx.wasPreRegexp || !regexpCloserAhead() {
			return false
		}
		return ctx.expectConst("/")
	}

	regexpClass := false
	regexpEnd := func() bool {
		if regexpClass {
			return false
		}
		return ctx.expectConst("/")
	}

	europeanDigitSeparator := func() bool {
		if !syntax.property(eur) {
			return false
		}
		if !isdigit(ctx.lastChar) {
			return false
		}
		peeked, err := ctx.rc.Peek(2)
		if err != nil || len(peeked) < 2 {
			return false
		}
		if peeked[0] != '\'' || !isdigit(peeked[1]) {
			return false
		}
		ctx.rc.Discard(1)
		if transdump {
			os.Stdout.Write([]byte{'\''})
		}
		ctx.updateCounters('\'')
		ctx.lastToken = "'"
		return true
	}

	escapeApplies := func() bool {
		switch mode {
		case stateINSTRING:
			if ctx.quoteDelims == nil {
				return false
			}
			stringDelims := ctx.quoteDelims[len(ctx.quoteDelims)-1]
			return hasPairFlag(stringDelims.spec, bse)
		case stateINREGEXP:
			return true
		default:
			return false
		}
	}

	consumePeekedLine := func() {
		ctx.rc.Discard(len(ctx.line))
		for _, c := range ctx.line {
			ctx.updateCounters(c)
		}
	}

	literateComment := func() bool {
		if !syntax.property(lit) || !ctx.wasNewline() {
			return false
		}
		peeked, err := ctx.rc.Peek(1)
		if err == nil && len(peeked) > 0 && peeked[0] == '>' {
			return false
		}
		line, ok := ctx.peekLine()
		if !ok {
			return false
		}
		ctx.line = append(ctx.line[:0], line...)
		consumePeekedLine()
		return true
	}

	for !ctx.eof() {
		if debug > 2 {
			ctx.breadcrumb("top of loop")
		}
		if (syntax.Lrb != "" || syntax.Lre != "" || syntax.property(heredoc)) && ctx.charConsumed == 0 {
			line, ok := ctx.peekLine()
			if !ok {
				break
			}
			ctx.line = append(ctx.line[:0], line...)

			if syntax.property(heredoc) {
				trimmed := trimLine(ctx.line)
				if ctx.heredoc != "" {
					if string(trimmed) == ctx.heredoc {
						ctx.heredoc = ""
					}
				} else {
					if hd := detectHeredoc(ctx.line); hd != "" {
						ctx.heredoc = hd
					}
				}
			}

			suppress := false
			if syntax.property(heredoc) && ctx.heredoc != "" {
				suppress = true
			}
			//if !suppress && syntax.Lrx != "" && ctx.matchline(syntax.Lrx) {
			//	suppress = true
			//}

			if ctx.lineRegion {
				if !suppress && syntax.Lre != "" && ctx.matchline(syntax.Lre) {
					ctx.lineRegion = false
				}
				consumePeekedLine()
				continue
			}

			if !suppress && syntax.Lrb != "" && ctx.matchline(syntax.Lrb) {
				ctx.lineRegion = true
				consumePeekedLine()
				continue
			}
		}
		// A backslash inside an escapable string disables whatever state
		// change the following character would otherwise have triggered,
		// including a closing quote delimiter.
		if ctx.expectConst("\\") {
			if escapeApplies() {
				if debug > 1 {
					ctx.breadcrumb("escaping...")
				}
				if ctx.expectConst("\n") {
					if debug > 1 {
						ctx.breadcrumb("backslash SLOC++")
					} else if transdump {
						dumpTransition("+")
					}
					stats.SLOC++
				} else {
					c, err := ctx.rc.ReadByte()
					if err != nil && err != io.EOF {
						panic("error while reading escaped character")
					}
					if err == nil {
						if debug > 1 {
							ctx.breadcrumb(fmt.Sprintf("discarding %q", []byte{c}))
						} else if transdump {
							os.Stdout.Write([]byte{c})
						}
						ctx.updateCounters(c)
					}
				}
				continue
			}
		}

		if mode == stateNORMAL {
			if ctx.expectConst(syntax.St) {
				flushNormalWord()
				ctx.wasPreRegexp = rxToken(syntax.St)
				ctx.markEligible()
				stats.LLOC++
				if debug > 1 {
					ctx.breadcrumb("lloc++")
				}
				continue
			} else if europeanDigitSeparator() {
				ctx.markEligible()
				if debug > 1 {
					ctx.breadcrumb("ignored digit separator apostrophe")
				}
				continue
			} else if len(syntax.She) > 0 && ctx.charCount == 0 && ctx.expectConst("#!") {
				/* we *don't* set slocEligible here! */
				if debug > 1 {
					ctx.breadcrumb("hashbang leader")
				}
				enterComment(delimSpec{Open: "#!", Close: "\n"})
				mode = stateCOMMENT
				ctx.saveMark()
				continue

			} else if syntax.property(lex) && ctx.wasNewline() && ctx.expectConst("%%\n") {
				ctx.lexsection++
				if debug > 1 {
					ctx.breadcrumb(fmt.Sprintf("lexsection = %d\n", ctx.lexsection))
				}
				continue
			} else if literateComment() {
				continue
			} else if ctx.lexsection != 1 && stringStart() {
				ctx.markEligible()
				if debug > 1 {
					ctx.breadcrumb(fmt.Sprintf("%q triggers NORMAL -> STRING", ctx.lastToken))
				} else if transdump {
					dumpTransition("STRING")
				}

				mode = stateINSTRING
				ctx.saveMark()
				continue
			} else if ctx.lexsection != 1 && commentStart(syntax) {
				/* we *don't* set slocEligible here! */
				if debug > 1 {
					ctx.breadcrumb(fmt.Sprintf("%q triggers NORMAL -> COMMENT", ctx.lastToken))
				} else if transdump {
					dumpTransition("COMMENT")
				}

				mode = stateCOMMENT
				ctx.saveMark()
				continue
			} else if ctx.lexsection != 1 && regexpStart() {
				ctx.markEligible()
				if debug > 1 {
					ctx.breadcrumb(fmt.Sprintf("%q triggers NORMAL -> REGEXP", ctx.lastToken))
				} else if transdump {
					dumpTransition("REGEXP")
				}

				mode = stateINREGEXP
				regexpClass = false
				ctx.wasPreRegexp = false
				ctx.saveMark()
				continue
			} else if syntax.property(end) && ctx.expectConst("__END__\n") {
				break
			} else if syntax.property(erlang) && ctx.expectConst("$\"") {
				continue
			}
		} else if mode == stateINSTRING {
			countline := ctx.slocEligible
			if stringEnd() {
				ctx.markEligible()
				if debug > 1 {
					ctx.breadcrumb(fmt.Sprintf("%q triggers STRING -> NORMAL", ctx.lastToken))
				} else if transdump {
					dumpTransition("NORMAL")
				}
				mode = stateNORMAL
				ctx.quoteDelims = nil
				ctx.newlineLatch = false
				ctx.wasPreRegexp = false
				continue
			} else if eol() {
				if countline {
					if debug > 1 {
						ctx.breadcrumb("string SLOC++")
					} else if transdump {
						dumpTransition("+")
					}
					stats.SLOC++
				}
				// We found a bare newline in a string without
				// preceding backslash.
				activeDelims := ctx.quoteDelims[len(ctx.quoteDelims)-1]
				if !hasPairFlag(activeDelims.spec, ml) && !ctx.newlineLatch {
					ctx.breadcrumb(fmt.Sprintf("WARNING - newline in %v string", activeDelims.spec))
					ctx.newlineLatch = true
				}

				// We COULD warn & reset mode to
				// "Normal", but lots of code does this,
				// so we'll just depend on the warning
				// for ending the program in a string to
				// catch syntactically erroneous
				// programs.
				continue
			}
		} else if mode == stateINREGEXP {
			if regexpEnd() {
				ctx.markEligible()
				if debug > 1 {
					ctx.breadcrumb(fmt.Sprintf("%q triggers REGEXP -> NORMAL", ctx.lastToken))
				} else if transdump {
					dumpTransition("NORMAL")
				}
				mode = stateNORMAL
				ctx.wasPreRegexp = false
				continue
			}
		} else { /* stateCOMMENT */
			if len(ctx.commentStack) == 0 {
				mode = stateNORMAL
				continue
			}
			active := ctx.commentStack[len(ctx.commentStack)-1]
			if active.Close == "\n" && ctx.hasPrefix("\n") {
				if debug > 1 {
					ctx.breadcrumb("newline triggers COMMENT->NORMAL")
				} else if transdump {
					dumpTransition("NORMAL")
				}
				ctx.commentStack = ctx.commentStack[:len(ctx.commentStack)-1]
				mode = stateNORMAL
				continue
			}
			if commentEnd() {
				// Bad grammar - ignore, we're not making an AST
				laststart := ctx.commentStack[len(ctx.commentStack)-1]
				nests := hasPairFlag(laststart, cnest)
				if ctx.lastToken != laststart.Close {
					if debug > 1 {
						ctx.breadcrumb(fmt.Sprintf("ignoring non-matching comment ender %q", ctx.lastToken))
					}
					continue
				}
				if nests {
					ctx.commentStack = ctx.commentStack[:len(ctx.commentStack)-1]
					if debug > 1 {
						ctx.breadcrumb(fmt.Sprintf("nesting comment leader decrements level to %d", len(ctx.commentStack)))
					}
				} else {
					ctx.commentStack = ctx.commentStack[:0]
					if debug > 1 {
						ctx.breadcrumb("non-nesting comment ender")
					}
				}
				if len(ctx.commentStack) == 0 {
					if debug > 1 {
						ctx.breadcrumb(fmt.Sprintf("%s triggers COMMENT->NORMAL", ctx.lastToken))
					} else if transdump {
						dumpTransition("NORMAL")
					}
					mode = stateNORMAL
				}
				continue
			} else if active.Close != "\n" && nestedCommentStart(syntax) {
				// New nested comment state is on the stack, nothing else to do
				continue
			}
		}

		countLine := ctx.slocEligible
		c, err := ctx.rc.ReadByte()
		if err != nil && err != io.EOF {
			panic("error while reading a character")
		}
		if debug > 1 {
			ctx.breadcrumb(fmt.Sprintf("state %s advancing on %q", stateNames[mode], []byte{c}))
		} else if transdump {
			os.Stdout.Write([]byte{c})
		}
		ctx.updateCounters(c)
		if !isspace(c) && (mode == stateNORMAL || mode == stateINSTRING || mode == stateINREGEXP) {
			if syntax.property(num) && strings.IndexByte("0123456789", c) > -1 {
				continue
			}
			ctx.markEligible()
		}
		if mode == stateNORMAL && len(rxTokens) > 0 {
			if c == '\n' {
				flushNormalWord()
			} else if isRegexpWordByte(c) {
				normalWord += string(c)
			} else if isspace(c) {
				flushNormalWord()
			} else {
				flushNormalWord()
				ctx.wasPreRegexp = rxToken(string(c))
			}
		} else if mode == stateINREGEXP {
			if c == '[' {
				regexpClass = true
			} else if c == ']' {
				regexpClass = false
			}
		}
		if c == '\n' && countLine {
			if debug > 1 {
				ctx.breadcrumb("SLOC++")
			} else if transdump {
				dumpTransition("+")
			}
			stats.SLOC++
		}
	}
	/* We're done with the file.  Handle EOF-without-EOL. */
	if ctx.slocEligible {
		if debug > 1 {
			ctx.breadcrumb("widow line SLOC++")
		} else if transdump {
			dumpTransition("+")
		}
		stats.SLOC++
	}

	if mode == stateCOMMENT {
		ctype := "comment"
		if len(ctx.commentStack) > 0 && ctx.commentStack[len(ctx.commentStack)-1].Close == "\n" {
			ctype = "trailing"
		}
		// The reason for this ugly guard is JavaScript without line breaks
		// embedded in HTML. The bare regexp element looks like a winged
		// comment. We could make this unnecessary by adding a new state.
		if !syntax.property(ktl) && ctx.lineNumber > 1 {
			ctx.breadcrumb(fmt.Sprintf("ERROR - terminated in %s comment beginning %d:%d", ctype, ctx.startline, ctx.startchar))
		}
	} else if mode == stateINSTRING {
		ctx.breadcrumb(fmt.Sprintf("ERROR - terminated in string beginning %d:%d", ctx.startline, ctx.startchar))
	} else if mode == stateINREGEXP {
		ctx.breadcrumb(fmt.Sprintf("ERROR - terminated in regexp beginning %d:%d", ctx.startline, ctx.startchar))
	}

	if syntax.property(num) {
		stats.LLOC = stats.SLOC
	}

	return []SourceStat{stats}
}

func goCounter(path string) (uint, bool) {
	var lloc uint

	content, err1 := os.ReadFile(path)
	if err1 != nil {
		return 0, false
	}

	fset := token.NewFileSet() // positions are relative to fset
	f, err2 := parser.ParseFile(fset, path, content, 0)
	if err2 != nil {
		return 0, false
	}

	// Inspect the AST and print all identifiers and literals.
	ast.Inspect(f, func(n ast.Node) bool {
		switch n.(type) {
		case *ast.AssignStmt: // sssignment or short variable declaration
			lloc++
		case *ast.BranchStmt: // break, continue, goto, or fallthrough
			lloc++
		case *ast.DeclStmt: // declaration in a statement list.
			lloc++
		case *ast.DeferStmt: // a defer statement.
			lloc++
		case *ast.ExprStmt: // stand-alone expression in a statement list.
			lloc++
		case *ast.GenDecl: // an import, constant, type or variable declaration
			lloc++
		case *ast.GoStmt: // go xxxx
			lloc++
		//case *ast.IfStmt:	// an if statement
		//	lloc++
		case *ast.ImportSpec: // package import line
			lloc++
		case *ast.IncDecStmt: // incement or decrement statement
			lloc++
		//case *ast.RangeStmt:	// for statement with a range clause.
		//	lloc++
		case *ast.ReturnStmt: // a return statement.
			lloc++
		//case *ast.SelectStmt:	// a select statement.
		//	lloc++
		case *ast.SendStmt: // a send statement.
			lloc++
			//case *ast.SwitchStmt:	// a switch statement.
			//	lloc++
		}
		// Not counted: BlockStmt, FuncDecl
		// Including IfStmt, RangeStmt, SelectStmt, SwitchStmt
		// is probably a better complexity metric, but no longer
		// strictly comparable with counting semis in C.
		return true
	})
	return lloc, true
}

// countGeneric - recognize lots of languages with generic syntax
func countGeneric(path string) []SourceStat {
	// Skip unreadable files early to avoid panics and misclassification.
	if f, err := os.Open(path); err != nil {
		return nil
	} else {
		f.Close()
	}

	ctx := new(countContext)
	var singleStat SourceStat
	singleStat.Path = path
	lopath := strings.ToLower(path)
	matchedExt := matchedExtensionSuffix(lopath)
	singleStat.cgi = strings.HasSuffix(lopath, ".cgi")

	autofilter := func(commentleaders []string) bool {
		if wasGeneratedAutomatically(ctx, path, commentleaders) {
			if listignores {
				fmt.Println(path)
				return true
			}
			if debug > 0 {
				fmt.Printf("automatic generation filter failed: %s\n", path)
			}
			return true
		}
		if debug > 0 {
			fmt.Printf("automatic generation filter passed: %s\n", path)
		}
		return false
	}

	tryCount := func(lang genericLanguage, trigger string, matchedByHashbang bool) ([]SourceStat, bool) {
		if trigger == "" {
			return nil, false
		}
		if autofilter(lineCommentLeaders(lang)) {
			return []SourceStat{singleStat}, true
		}
		if !passesTextChecks(ctx, path, lang.Name, lang.Require, lang.Forbid) {
			return nil, false
		}
		stats := genericCounter(ctx, path, lang, trigger)
		if stats == nil {
			return nil, false
		}
		if len(lang.Require) > 0 {
			trackStringMatch(path, lang.Name)
		}
		for j := range stats {
			stats[j].contentMatch = matchedByHashbang || len(lang.Require) > 0
			stats[j].cgi = singleStat.cgi
		}
		if strings.HasSuffix(path, ".go") {
			if len(stats) > 0 {
				if lloc, ok := goCounter(path); ok || lloc > 0 {
					stats[0].LLOC = lloc
				}
			}
		}
		return stats, true
	}

	allowNonExecutableHashbang := singleStat.cgi || matchedExt != ""
	for i := range genericLanguages {
		lang := genericLanguages[i]
		if len(lang.She) == 0 || !hashbang(ctx, path, lang.She, allowNonExecutableHashbang) {
			continue
		}
		trigger := "#"
		if singleStat.cgi {
			trigger = ".cgi"
		}
		if stats, ok := tryCount(lang, trigger, true); ok {
			return stats
		}
	}

	for i := range genericLanguages {
		lang := genericLanguages[i]
		var trigger string
		if !singleStat.cgi {
			trigger = lang.pathMatches(lopath, matchedExt)
		}
		if stats, ok := tryCount(lang, trigger, false); ok {
			return stats
		}
	}

	// Without this fallthrough to returning an empty stat block,
	// we'd get no report on unclassifiables.
	return []SourceStat{singleStat}
}

func recountAsLanguage(original SourceStat, language string, ext string) (SourceStat, bool) {
	syntax, ok := syntaxForLanguageExtension(language, ext)
	if !ok {
		return original, false
	}
	ctx := new(countContext)
	stats := genericCounter(ctx, original.Path, syntax, ext)
	if len(stats) == 0 {
		return original, false
	}
	reassigned := stats[0]
	reassigned.cgi = original.cgi
	reassigned.contentMatch = false
	if strings.HasSuffix(original.Path, ".go") {
		if lloc, ok := goCounter(original.Path); ok || lloc > 0 {
			reassigned.LLOC = lloc
		}
	}
	return reassigned, true
}

func inferUniqueContentMatchedTypes(stats []SourceStat, enabled bool) {
	if !enabled {
		return
	}
	assignments := uniqueStringMatchAssignments()
	if len(assignments) == 0 {
		return
	}
	for i := range stats {
		if stats[i].contentMatch {
			continue
		}
		ext := matchedExtensionSuffix(stats[i].Path)
		if ext == "" || !sharedLanguageExtension(ext) {
			continue
		}
		language, ok := assignments[ext]
		if !ok {
			continue
		}
		reassigned, ok := recountAsLanguage(stats[i], language, ext)
		if !ok {
			continue
		}
		stats[i] = reassigned
	}
}

func isDirectory(path string) bool {
	fileInfo, err := os.Lstat(path)
	return err == nil && fileInfo.Mode().IsDir()
}

func isRegular(path string) bool {
	fileInfo, err := os.Lstat(path)
	return err == nil && fileInfo.Mode().IsRegular()
}

func excludedBasenames() []string {
	names := make([]string, 0, len(neverInterestingByBasename))
	for name := range neverInterestingByBasename {
		names = append(names, name)
	}
	sort.Strings(names)
	return names
}

func walkDirectories(directories []string, noGitmodules bool, fileQueue chan<- string) {
	if len(directories) == 0 {
		return
	}
	fileListQueue := make(chan *gocodewalker.File, runtime.NumCPU())
	fileWalker := gocodewalker.NewParallelFileWalker(directories, fileListQueue)
	fileWalker.SetConcurrency(runtime.NumCPU())
	fileWalker.IncludeHidden = true
	fileWalker.IgnoreGitModules = noGitmodules
	fileWalker.ExcludeDirectory = excludedBasenames()
	fileWalker.LocationExcludePattern = append(fileWalker.LocationExcludePattern, neverInterestingByInfix...)
	fileWalker.SetErrorHandler(func(err error) bool {
		if debug > 0 {
			fmt.Fprintln(os.Stderr, err)
		}
		return true
	})

	errc := make(chan error, 1)
	go func() {
		errc <- fileWalker.Start()
	}()

	for file := range fileListQueue {
		fileQueue <- file.Location
	}
	if err := <-errc; err != nil && debug > 0 {
		fmt.Fprintln(os.Stderr, err)
	}
}

// repoDriver isolates version-control-specific repository detection
// and file enumeration. To add another VCS, define a repoRoot hook
// and a listFiles hook, then append a driver here.
type repoDriver struct {
	name      string
	repoRoot  func(path string, isDir bool) (string, error)
	listFiles func(repoRoot string, root string) ([]string, error)
}

type repoMatch struct {
	driver   repoDriver
	repoRoot string
}

type svnInfoXML struct {
	Entry struct {
		WCInfo struct {
			WCRootAbsPath string `xml:"wcroot-abspath"`
		} `xml:"wc-info"`
	} `xml:"entry"`
}

type svnStatusXML struct {
	Targets []struct {
		Entries []struct {
			Path     string `xml:"path,attr"`
			WCStatus struct {
				Item string `xml:"item,attr"`
			} `xml:"wc-status"`
		} `xml:"entry"`
	} `xml:"target"`
}

type p4TaggedRecord map[string]string

func gitRepoRoot(path string, isDir bool) (string, error) {
	probe := path
	if !isDir {
		probe = filepath.Dir(path)
	}
	out, err := exec.Command("git", "-C", probe, "rev-parse", "--show-toplevel").Output()
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(out)), nil
}

func gitTrackedFiles(repoRoot string, root string) ([]string, error) {
	relRoot, err := filepath.Rel(repoRoot, root)
	if err != nil {
		return nil, err
	}
	if relRoot == ".." || strings.HasPrefix(relRoot, ".."+string(os.PathSeparator)) {
		return nil, fmt.Errorf("%s is not inside %s", root, repoRoot)
	}
	out, err := exec.Command("git", "-C", repoRoot, "ls-files", "-z", "--", filepath.ToSlash(relRoot)).Output()
	if err != nil {
		return nil, err
	}
	paths := make([]string, 0)
	for _, entry := range bytes.Split(out, []byte{0}) {
		if len(entry) == 0 {
			continue
		}
		paths = append(paths, filepath.Join(repoRoot, filepath.FromSlash(string(entry))))
	}
	return paths, nil
}

func svnWorkingCopyRoot(path string, isDir bool) (string, error) {
	probe := path
	if !isDir {
		probe = filepath.Dir(path)
	}
	out, err := exec.Command("svn", "info", "--xml", probe).Output()
	if err != nil {
		return "", err
	}
	var info svnInfoXML
	if err := xml.Unmarshal(out, &info); err != nil {
		return "", err
	}
	if info.Entry.WCInfo.WCRootAbsPath == "" {
		return "", fmt.Errorf("svn info returned no working-copy root for %s", probe)
	}
	return strings.TrimSpace(info.Entry.WCInfo.WCRootAbsPath), nil
}

func svnTrackedFiles(repoRoot string, root string) ([]string, error) {
	out, err := exec.Command("svn", "status", "-v", "--xml", root).Output()
	if err != nil {
		return nil, err
	}
	var status svnStatusXML
	if err := xml.Unmarshal(out, &status); err != nil {
		return nil, err
	}
	paths := make([]string, 0)
	for _, target := range status.Targets {
		for _, entry := range target.Entries {
			switch entry.WCStatus.Item {
			case "unversioned", "ignored", "external", "none":
				continue
			}
			paths = append(paths, canonicalPath(entry.Path))
		}
	}
	return paths, nil
}

func hgRepoRoot(path string, isDir bool) (string, error) {
	probe := path
	if !isDir {
		probe = filepath.Dir(path)
	}
	out, err := exec.Command("hg", "--cwd", probe, "root").Output()
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(out)), nil
}

func hgTrackedFiles(repoRoot string, root string) ([]string, error) {
	relRoot, err := filepath.Rel(repoRoot, root)
	if err != nil {
		return nil, err
	}
	if relRoot == ".." || strings.HasPrefix(relRoot, ".."+string(os.PathSeparator)) {
		return nil, fmt.Errorf("%s is not inside %s", root, repoRoot)
	}
	pathArg := filepath.ToSlash(relRoot)
	out, err := exec.Command("hg", "--cwd", repoRoot, "files", "-0", pathArg).CombinedOutput()
	if err != nil {
		if len(out) == 0 {
			return []string{}, nil
		}
		return nil, err
	}
	paths := make([]string, 0)
	for _, entry := range bytes.Split(out, []byte{0}) {
		if len(entry) == 0 {
			continue
		}
		paths = append(paths, filepath.Join(repoRoot, filepath.FromSlash(string(entry))))
	}
	return paths, nil
}

func parseP4TaggedRecords(out []byte) []p4TaggedRecord {
	records := make([]p4TaggedRecord, 0)
	var current p4TaggedRecord
	lines := strings.Split(string(out), "\n")
	for _, line := range lines {
		line = strings.TrimRight(line, "\r")
		if line == "" {
			if current != nil {
				records = append(records, current)
				current = nil
			}
			continue
		}
		if !strings.HasPrefix(line, "... ") {
			continue
		}
		payload := strings.TrimPrefix(line, "... ")
		parts := strings.SplitN(payload, " ", 2)
		if len(parts) != 2 {
			continue
		}
		if current == nil {
			current = make(p4TaggedRecord)
		} else if _, exists := current[parts[0]]; exists {
			records = append(records, current)
			current = make(p4TaggedRecord)
		}
		current[parts[0]] = parts[1]
	}
	if current != nil {
		records = append(records, current)
	}
	return records
}

func p4RepoRoot(path string, isDir bool) (string, error) {
	probe := path
	if !isDir {
		probe = filepath.Dir(path)
	}
	out, err := exec.Command("p4", "-d", probe, "info").CombinedOutput()
	if err != nil {
		return "", err
	}
	lines := strings.Split(string(out), "\n")
	for _, line := range lines {
		line = strings.TrimSpace(line)
		if line == "Client unknown." {
			return "", fmt.Errorf("p4 client unknown for %s", probe)
		}
		if strings.HasPrefix(line, "Client root:") {
			root := strings.TrimSpace(strings.TrimPrefix(line, "Client root:"))
			if root == "" {
				break
			}
			if strings.EqualFold(root, "null") {
				return canonicalPath(probe), nil
			}
			return root, nil
		}
	}
	return "", fmt.Errorf("p4 info returned no client root for %s", probe)
}

func p4FileSpec(root string, isDir bool) string {
	if !isDir {
		return root
	}
	return filepath.Join(root, "...")
}

var p4HavePath = regexp.MustCompile(`^.*#[0-9]+ - (.*)$`)

func parseP4HavePaths(out []byte) []string {
	paths := make([]string, 0)
	lines := strings.Split(string(out), "\n")
	for _, line := range lines {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}
		matches := p4HavePath.FindStringSubmatch(line)
		if len(matches) == 2 {
			paths = append(paths, matches[1])
		}
	}
	return paths
}

func appendUniquePaths(dst []string, seen map[string]struct{}, src []string) []string {
	for _, path := range src {
		canonical := canonicalPath(path)
		if _, exists := seen[canonical]; exists {
			continue
		}
		seen[canonical] = struct{}{}
		dst = append(dst, canonical)
	}
	return dst
}

func p4MapDepotPaths(repoRoot string, depotPaths []string) ([]string, error) {
	if len(depotPaths) == 0 {
		return []string{}, nil
	}
	const batchSize = 128
	paths := make([]string, 0, len(depotPaths))
	for start := 0; start < len(depotPaths); start += batchSize {
		end := start + batchSize
		if end > len(depotPaths) {
			end = len(depotPaths)
		}
		args := []string{"-d", repoRoot, "-ztag", "where"}
		args = append(args, depotPaths[start:end]...)
		out, err := exec.Command("p4", args...).CombinedOutput()
		if err != nil {
			if len(bytes.TrimSpace(out)) == 0 {
				continue
			}
			return nil, err
		}
		for _, record := range parseP4TaggedRecords(out) {
			if path, ok := record["path"]; ok && path != "" {
				paths = append(paths, path)
			}
		}
	}
	return paths, nil
}

func p4TrackedFiles(repoRoot string, root string) ([]string, error) {
	fi, err := os.Stat(root)
	if err != nil {
		return nil, err
	}
	fileSpec := p4FileSpec(root, fi.Mode().IsDir())

	haveOut, err := exec.Command("p4", "-d", repoRoot, "have", fileSpec).CombinedOutput()
	if err != nil && len(bytes.TrimSpace(haveOut)) != 0 {
		return nil, err
	}
	paths := parseP4HavePaths(haveOut)
	seen := make(map[string]struct{}, len(paths))
	paths = appendUniquePaths([]string{}, seen, paths)

	openedOut, err := exec.Command("p4", "-d", repoRoot, "-ztag", "opened", fileSpec).CombinedOutput()
	if err != nil && len(bytes.TrimSpace(openedOut)) != 0 {
		return nil, err
	}
	openedDepotPaths := make([]string, 0)
	for _, record := range parseP4TaggedRecords(openedOut) {
		if depotPath, ok := record["depotFile"]; ok && depotPath != "" {
			openedDepotPaths = append(openedDepotPaths, depotPath)
		}
	}
	openedPaths, err := p4MapDepotPaths(repoRoot, openedDepotPaths)
	if err != nil {
		return nil, err
	}
	paths = appendUniquePaths(paths, seen, openedPaths)
	return paths, nil
}

var repoDrivers = []repoDriver{
	{name: "Git", repoRoot: gitRepoRoot, listFiles: gitTrackedFiles},
	{name: "Mercurial", repoRoot: hgRepoRoot, listFiles: hgTrackedFiles},
	{name: "Perforce", repoRoot: p4RepoRoot, listFiles: p4TrackedFiles},
	{name: "Subversion", repoRoot: svnWorkingCopyRoot, listFiles: svnTrackedFiles},
}

func canonicalPath(path string) string {
	resolved, err := filepath.EvalSymlinks(path)
	if err == nil {
		return resolved
	}
	return path
}

func isWithinRoot(root string, path string) bool {
	rel, err := filepath.Rel(root, path)
	if err != nil {
		return false
	}
	return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator)))
}

func rootDisplayPath(root string, cwd string, absPath string) string {
	if filepath.IsAbs(root) {
		return absPath
	}
	rel, err := filepath.Rel(cwd, absPath)
	if err != nil {
		return absPath
	}
	return rel
}

// When multiple VCS roots match, prefer the deepest one so nested
// working copies win over enclosing repositories.
func detectRepository(path string, isDir bool) *repoMatch {
	bestDepth := -1
	var best *repoMatch
	for i := range repoDrivers {
		repoRoot, err := repoDrivers[i].repoRoot(path, isDir)
		if err != nil {
			continue
		}
		repoRoot = canonicalPath(repoRoot)
		if !isWithinRoot(repoRoot, path) {
			continue
		}
		depth := strings.Count(repoRoot, string(os.PathSeparator))
		if best == nil || depth > bestDepth {
			match := repoMatch{
				driver:   repoDrivers[i],
				repoRoot: repoRoot,
			}
			best = &match
			bestDepth = depth
		}
	}
	return best
}

func emitRepoTracked(root string, fi os.FileInfo, cwd string, fileQueue chan<- string) bool {
	absRoot, err := filepath.Abs(root)
	if err != nil {
		return false
	}
	absRoot = canonicalPath(absRoot)
	match := detectRepository(absRoot, fi.Mode().IsDir())
	if match == nil {
		return false
	}
	paths, err := match.driver.listFiles(match.repoRoot, absRoot)
	if err != nil {
		if debug > 0 {
			fmt.Fprintf(os.Stderr, "%s repository listing failed: %v\n", match.driver.name, err)
		}
		return false
	}
	for _, path := range paths {
		fileQueue <- rootDisplayPath(root, cwd, canonicalPath(path))
	}
	return true
}

func walkRoots(roots []string, keepwalk bool, noGitmodules bool, workerCount int) {
	if workerCount < 1 {
		workerCount = 1
	}

	fileQueue := make(chan string, workerCount)
	var wg sync.WaitGroup
	for i := 0; i < workerCount; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for path := range fileQueue {
				filter(path)
			}
		}()
	}
	defer func() {
		close(fileQueue)
		wg.Wait()
	}()

	cwd, err := os.Getwd()
	if err != nil {
		cwd = "."
	}
	cwd = canonicalPath(cwd)

	directories := make([]string, 0, len(roots))
	for i := range roots {
		fi, err := os.Stat(roots[i])
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}
		if !keepwalk && emitRepoTracked(roots[i], fi, cwd, fileQueue) {
			continue
		}
		if fi.Mode().IsDir() {
			directories = append(directories, roots[i])
			continue
		}
		fileQueue <- roots[i]
	}
	walkDirectories(directories, noGitmodules, fileQueue)
}

const nulScanBytes = 1024

func hasTraitExtension(path string) bool {
	lopath := strings.ToLower(path)
	if matchedExtensionSuffix(lopath) != "" {
		return true
	}
	for i := range genericLanguages {
		if genericLanguages[i].fileMatches(lopath) != "" {
			return true
		}
	}
	return false
}

func hasNeverInterestingSuffix(path string) bool {
	lopath := strings.ToLower(path)
	for suffix := range neverInterestingBySuffix {
		if strings.HasSuffix(lopath, strings.ToLower(suffix)) {
			return true
		}
	}
	return false
}

func inWafBuildDir(path string) bool {
	for probe := filepath.Clean(path); ; probe = filepath.Dir(probe) {
		if filepath.Base(probe) == "build" && isDirectory(probe) && isRegular(filepath.Join(filepath.Dir(probe), "waf")) {
			return true
		}
		next := filepath.Dir(probe)
		if next == probe {
			return false
		}
	}
}

func hasNULPrefix(path string, limit int) bool {
	f, err := os.Open(path)
	if err != nil {
		return false
	}
	defer f.Close()

	buf := make([]byte, limit)
	n, err := f.Read(buf)
	if err != nil && err != io.EOF {
		return false
	}
	for _, b := range buf[:n] {
		if b == 0 {
			return true
		}
	}
	return false
}

// filter() winnows out uninteresting paths before handing them to process
func filter(path string) error {
	if debug > 0 {
		fmt.Printf("entering filter: %s\n", path)
	}
	if hasNeverInterestingSuffix(path) {
		if debug > 0 {
			fmt.Printf("suffix filter failed: %s\n", path)
		}
		return nil
	}
	for i := range neverInterestingByPrefix {
		var interesting bool = false
		if strings.HasPrefix(path, neverInterestingByPrefix[i]) {
			for j := range actuallyInterestingByPrefix {
				if strings.HasPrefix(path, actuallyInterestingByPrefix[j]) {
					interesting = true
					break
				}
			}
			if !interesting {
				if debug > 0 {
					fmt.Printf("prefix filter failed: %s\n", path)
				}
				return nil
			}
		}
	}
	for i := range neverInterestingByInfix {
		if strings.Contains(path, neverInterestingByInfix[i]) {
			if debug > 0 {
				fmt.Printf("infix filter failed: %s\n", path)
			}
			if isDirectory(path) {
				if debug > 0 {
					fmt.Printf("directory skipped: %s\n", path)
				}
				return filepath.SkipDir
			}
			return nil
		}
	}
	basename := filepath.Base(path)
	if neverInterestingByBasename[strings.ToLower(basename)] {
		if debug > 0 {
			fmt.Printf("basename filter failed: %s\n", path)
		}
		return nil
	}
	if !hasTraitExtension(path) && isRegular(path) && hasNULPrefix(path, nulScanBytes) {
		if debug > 0 {
			fmt.Printf("nul filter failed: %s\n", path)
		}
		return nil
	}
	if exclusions != nil && exclusions.MatchString(path) {
		if debug > 0 {
			fmt.Printf("exclusion '%s' filter failed: %s\n", exclusions, path)
		}
		return nil
	}
	if inWafBuildDir(path) {
		if debug > 0 {
			fmt.Printf("waf-build filter failed: %s\n", path)
		}
		if isDirectory(path) {
			return filepath.SkipDir
		}
		return nil
	}

	/* has to come after the infix check for directory */
	if !isRegular(path) {
		if debug > 0 {
			fmt.Printf("regular-file filter failed: %s\n", path)
		}
		return nil
	}

	/* toss generated Makefiles */
	if basename == "Makefile" {
		if _, err := os.Stat(path + ".in"); err == nil {
			if debug > 0 {
				fmt.Printf("generated-makefile filter failed: %s\n", path)
			}
			return nil
		}
	}

	if debug > 0 {
		fmt.Printf("passed filter: %s\n", path)
	}

	// Now the real work gets done
	for _, st := range countGeneric(path) {
		pipeline <- st
	}

	return nil
}

type countRecord struct {
	language   string
	slinecount uint
	llinecount uint
	filecount  uint
	cgi        bool
}

func (r countRecord) reportLanguage() string {
	return displayLanguage(r.language, r.cgi)
}

func reassignCHeaderCounts(counts map[string]countRecord) {
	headers := counts["C-header"]
	if headers.slinecount == 0 {
		return
	}
	for i := range cHeaderPriority {
		language := cHeaderPriority[i]
		if counts[language].slinecount == 0 {
			continue
		}
		tmp := counts[language]
		tmp.slinecount += headers.slinecount
		tmp.llinecount += headers.llinecount
		tmp.filecount += headers.filecount
		counts[language] = tmp
		delete(counts, "C-header")
		break
	}
}

// cocomo81() performs old-style COCOMO estimation
func cocomo81(sloc uint) float64 {
	const cTIMEMULT = 2.4
	const cTIMEEXP = 1.05
	fmt.Printf("\nTotal Physical Source Lines of Code (SLOC)                = %d\n", sloc)
	fmt.Printf(" (COCOMO I model, Person-Months = %2.2f * (KSLOC**%2.2f))\n", cTIMEMULT, cTIMEEXP)
	return cTIMEMULT * math.Pow(float64(sloc)/1000, cTIMEEXP)
}

// cocomo2000(): see https://en.wikipedia.org/wiki/COCOMO
func cocomo2000(lloc uint) float64 {
	const cTIMEMULT = 3.2
	const cTIMEEXP = 1.05
	fmt.Printf("\nTotal Logical Source Lines of Code (LLOC)                 = %d\n", lloc)
	fmt.Printf(" (COCOMO II model, Person-Months = %2.2f * (KLOC**%2.2f))\n", cTIMEMULT, cTIMEEXP)
	return cTIMEMULT * math.Pow(float64(lloc)/1000, cTIMEEXP)
}

func reportCocomo(loc uint, curve func(uint) float64) {
	const cSCHEDMULT = 2.5
	const cSCHEDEXP = 0.38
	const cOVERHEAD = 2.40
	personMonths := curve(loc)
	fmt.Printf("Development Effort Estimate, Person-Years (Person-Months) = %2.2f (%2.2f)\n", personMonths/12, personMonths)
	schedMonths := cSCHEDMULT * math.Pow(personMonths, cSCHEDEXP)
	fmt.Printf("Schedule Estimate, Years (Months)                         = %2.2f (%2.2f)\n", schedMonths/12, schedMonths)
	fmt.Printf(" (COCOMO model, Months = %2.2f * (person-months**%2.2f))\n", cSCHEDMULT, cSCHEDEXP)
	fmt.Printf("Estimated Average Number of Developers (Effort/Schedule)  = %2.2f\n", personMonths/schedMonths)
	fmt.Printf("Total Estimated Cost to Develop                           = $%d\n", int(cSALARY*(personMonths/12)*cOVERHEAD))
	fmt.Printf(" (average salary = $%d/year, overhead = %2.2f).\n", cSALARY, cOVERHEAD)
}

// listLanguages lists all languages for which we can extract line counts.
// It also performs a sanity check on identifying file extensions and
// interpreter names.
func listLanguages(lloc bool, nodocs bool) ([]string, bool) {
	names := []string{"Perl", "Raku"}
	suffixCounts := make(map[string]int)
	nameCounts := make(map[string]int)
	duplicates := false
	for i := range genericLanguages {
		lang := genericLanguages[i]
		if lang.property(ign) {
			continue
		}
		nameCounts[lang.Name]++
		if len(lang.Require) == 0 && len(lang.Forbid) == 0 {
			for _, s := range lang.Ext {
				suffixCounts[s]++
			}
		}
		for _, s := range lang.Ext {
			if suffixCounts[s] > 1 {
				fmt.Fprintf(os.Stderr, "loccount: extension %s duplicated\n", s)
				duplicates = true
			}
		}
		if nameCounts[lang.Name] == 1 {
			if nodocs && lang.property(doc) {
				continue
			}
			if !lloc || len(genericLanguages[i].St) > 0 {
				names = append(names, lang.Name)
			}
		}
	}

	sort.Strings(names)
	return names, duplicates
}

func listExtensions() {
	extensions := map[string][]string{
		"python": {".py"},
		"waf":    {"waf"},
		"perl":   {"pl", "pm"},
	}
	for i := range genericLanguages {
		lang := genericLanguages[i]
		for _, s := range lang.Ext {
			extensions[lang.Name] = append(extensions[lang.Name], s)
		}
	}

	names, duplicates := listLanguages(false, false)
	for i := range names {
		fmt.Printf("%s: %v\n", names[i], extensions[names[i]])
	}
	if duplicates {
		os.Exit(1)
	}
}

// How to describe a language

type quirkInfo struct {
	bit   uint
	label string
}

var specialQuirks = []quirkInfo{
	{eur, "an apostrophe between digits is treated as a numeric separator"},
	{end, "everything after __END__ is treated as comment text"},
	{erlang, "'$' escapes a following double quote"},
	{f77, "the language uses Fortran 77 comment syntax"},
	{f90, "the language uses Fortran 90 comment syntax"},
	{heredoc, "the language supports Perl-style heredoc delimiters that suppress line-region comment transitions while active"},
	{ktl, "line comments ending at EOF without a final newline are accepted"},
	{lex, "the language has Lex-style %% sections"},
	{lit, "the language supports Haskell-style literate syntax"},
	{num, "line numbers are ignored"},
	{outdent, "some multiline strings terminate by indentation outdent rather than by a fixed closer"},
	{tcl, "a string starts only when preceded by punctuation or whitespace"},
	{yaml, "the language uses YAML line and block-scalar counting"},
	{xcpp, "the language uses the C preprocessor"},
	{xlsbc, "a leading triple-quoted string is counted as a comment"},
	{xssep, "the language uses statement separators rather than terminators"},
	{xtsmc, "module comments are triple-quoted strings"},
}

func hasQuirk(bits uint, flag uint) bool {
	return bits&flag != 0
}

func quote(s string) string {
	return "'" + s + "'"
}

func joinEnglish(items []string, conj string) string {
	switch len(items) {
	case 0:
		return ""
	case 1:
		return items[0]
	case 2:
		return items[0] + " " + conj + " " + items[1]
	default:
		return strings.Join(items[:len(items)-1], ", ") + ", " + conj + " " + items[len(items)-1]
	}
}

func orderedLabels(bits uint, source map[int]string) []string {
	keys := make([]int, 0, len(source))
	for key := range source {
		keys = append(keys, key)
	}
	sort.Ints(keys)
	labels := make([]string, 0, len(keys))
	for _, key := range keys {
		if hasQuirk(bits, uint(key)) {
			labels = append(labels, source[key])
		}
	}
	return labels
}

func describePairQuirks(part delimSpec) string {
	var bits []string
	if hasPairFlag(part, cnest) {
		bits = append(bits, "nested")
	}
	if hasPairFlag(part, sol) {
		bits = append(bits, "start-of-line only")
	}
	if hasPairFlag(part, ind) {
		bits = append(bits, "may be indented")
	}
	if hasPairFlag(part, bse) {
		bits = append(bits, "backslash-escapable")
	}
	if hasPairFlag(part, re) {
		bits = append(bits, "regexp opener")
	}
	if hasPairFlag(part, ml) {
		bits = append(bits, "multiline")
	}
	if hasPairFlag(part, outdent) {
		bits = append(bits, "outdent-terminated")
	}
	if len(bits) == 0 {
		return ""
	}
	return " (" + joinEnglish(bits, "and") + ")"
}

type selectorReportFormat int

const (
	selectorReportPlain selectorReportFormat = iota
	selectorReportHTML
)

type reportScope string

const (
	reportScopeSelection reportScope = "selection"
	reportScopeAll       reportScope = "all"
)

type namedLink struct {
	label string
	url   string
}

type languageVariantReport struct {
	name   string
	traits []string
}

type languageReport struct {
	name        string
	aliases     []string
	description string
	links       []namedLink
	variants    []languageVariantReport
}

func traitDescriptions(lang genericLanguage) []string {
	var parts []string

	// Opening sentence
	classtags := orderedLabels(lang.Fl, classes)
	if len(classtags) > 0 {
		parts = append(parts, fmt.Sprintf("This language is a %s.", joinEnglish(classtags, "and")))
	}

	statustags := orderedLabels(lang.Fl, status)
	if len(statustags) > 0 {
		parts = append(parts, fmt.Sprintf("It is %s.", joinEnglish(statustags, "and")))
	}

	// Extensions
	if len(lang.Ext) > 0 {
		exts := make([]string, 0, len(lang.Ext))
		for _, ext := range lang.Ext {
			exts = append(exts, quote(ext))
		}
		if len(exts) == 1 {
			parts = append(parts, fmt.Sprintf("Recognized by this pathname suffix: %s.", exts[0]))
		} else {
			parts = append(parts, fmt.Sprintf("Recognized by these pathname suffixes: %s.", joinEnglish(exts, "and")))
		}
	}
	// Files
	if len(lang.Files) > 0 {
		files := make([]string, 0, len(lang.Files))
		for _, file := range lang.Files {
			files = append(files, quote(file))
		}
		if len(files) == 1 {
			parts = append(parts, fmt.Sprintf("Recognized by this exact basename: %s.", files[0]))
		} else {
			parts = append(parts, fmt.Sprintf("Recognized by these exact basenames: %s.", joinEnglish(files, "and")))
		}
	}
	if len(lang.She) > 0 {
		interpreters := make([]string, 0, len(lang.She))
		for _, interp := range lang.She {
			interpreters = append(interpreters, quote(interp))
		}
		if len(interpreters) == 1 {
			parts = append(parts, fmt.Sprintf("Recognized by shebang interpreter name: %s.", interpreters[0]))
		} else {
			parts = append(parts, fmt.Sprintf("Recognized by shebang interpreter names %s.", joinEnglish(interpreters, "and")))
		}
	}

	// Comment syntax
	var commentBits []string
	if len(lang.Bc) > 0 {
		linePairs := make([]string, 0, len(lang.Bc))
		blockPairs := make([]string, 0, len(lang.Bc))
		for _, pair := range lang.Bc {
			if pair.Close == "\n" {
				desc := fmt.Sprintf("%s%s", quote(pair.Open), describePairQuirks(pair))
				linePairs = append(linePairs, desc)
			} else {
				desc := fmt.Sprintf("%s ... %s%s", quote(pair.Open), quote(pair.Close), describePairQuirks(pair))
				blockPairs = append(blockPairs, desc)
			}
		}
		if len(linePairs) > 0 {
			commentBits = append(commentBits, "line comments use "+joinEnglish(linePairs, "or"))
		}
		if len(blockPairs) > 0 {
			commentBits = append(commentBits, "block comments use "+joinEnglish(blockPairs, "or"))
		}
	}
	if len(commentBits) > 0 {
		parts = append(parts, "Comment syntax: "+joinEnglish(commentBits, "and")+".")
	}
	if lang.Lrb != "" || lang.Lre != "" { // || lag.LrX != ""
		var regionBits []string
		if lang.Lrb != "" {
			regionBits = append(regionBits, fmt.Sprintf("non-code regions begin on lines matching %s", quote(lang.Lrb)))
		}
		if lang.Lre != "" {
			regionBits = append(regionBits, fmt.Sprintf("non-code regions end on lines matching %s", quote(lang.Lre)))
		}
		//if lang.Lrx != "" {
		//	regionBits = append(regionBits, fmt.Sprintf("line-region transitions are suppressed on lines matching %s", quote(lang.Lrx)))
		//}
		if len(regionBits) > 0 {
			parts = append(parts, "Line-region syntax: "+joinEnglish(regionBits, "and")+".")
		}
	}

	// Strings and multiline literals
	var stringBits []string
	if len(lang.Str) > 0 {
		delims := make([]string, 0, len(lang.Str))
		for _, pair := range lang.Str {
			delims = append(delims, fmt.Sprintf("%s ... %s%s", quote(pair.Open), quote(pair.Close), describePairQuirks(pair)))
		}
		stringBits = append(stringBits, "string literals delimited by "+joinEnglish(delims, "and"))
	}
	if len(stringBits) > 0 {
		parts = append(parts, "For strings and literals, it supports "+joinEnglish(stringBits, "and")+".")
	}

	// Statement syntax
	var stmtBits []string
	if lang.St != "" {
		stmtBits = append(stmtBits, fmt.Sprintf("the statement terminator %s", quote(lang.St)))
	}
	if lang.Sb != "" {
		stmtBits = append(stmtBits, fmt.Sprintf("statement blocks beginning with %s", quote(lang.Sb)))
	}
	if lang.Eb != "" {
		stmtBits = append(stmtBits, fmt.Sprintf("statement blocks ending with %s", quote(lang.Eb)))
	}
	if hasQuirk(lang.Fl, xssep) {
		stmtBits = append(stmtBits, "statement separators rather than terminators")
	}
	if len(stmtBits) > 0 {
		parts = append(parts, "Its statement syntax uses "+joinEnglish(stmtBits, "and")+".")
	}

	// Other Fl
	// FIXME: Report Lrb, Lre, Lrx here
	var otherBits []string
	for _, q := range specialQuirks {
		// xssep is already mentioned with statement syntax; skip duplicate phrasing there if desired.
		if q.bit == xssep && (lang.St != "" || lang.Sb != "" || lang.Eb != "") {
			continue
		}
		if hasQuirk(lang.Fl, q.bit) {
			otherBits = append(otherBits, q.label)
		}
	}
	if len(otherBits) > 0 {
		parts = append(parts, "Quirks: "+strings.Join(otherBits, "; ")+".")
	}

	// Recognition rules
	if len(lang.Require) > 0 {
		reqs := make([]string, 0, len(lang.Require))
		for _, r := range lang.Require {
			reqs = append(reqs, quote(r))
		}
		parts = append(parts, "To recognize a file as this language, every one of these regexes must match at least one line: "+joinEnglish(reqs, "and")+".")
	}
	if len(lang.Forbid) > 0 {
		forbids := make([]string, 0, len(lang.Forbid))
		for _, r := range lang.Forbid {
			forbids = append(forbids, quote(r))
		}
		parts = append(parts, "Recognition fails if any line matches one of these regexes: "+joinEnglish(forbids, "and")+".")
	}

	return parts
}

func languageMatchesSelector(selector *regexp.Regexp, key string, aka string) bool {
	if selector == nil {
		return true
	}
	if selector.MatchString(key) {
		return true
	}
	for _, alias := range strings.Split(aka, "\n") {
		if alias != "" && selector.MatchString(alias) {
			return true
		}
	}
	return false
}

func parseAliases(aka string) []string {
	if aka == "" {
		return nil
	}
	aliases := make([]string, 0)
	for _, alias := range strings.Split(aka, "\n") {
		if alias != "" {
			aliases = append(aliases, alias)
		}
	}
	return aliases
}

func referenceLinks(links map[string]string) []namedLink {
	if len(links) == 0 {
		return nil
	}
	labels := make([]string, 0, len(links))
	for label := range links {
		labels = append(labels, label)
	}
	sort.Strings(labels)

	parts := make([]namedLink, 0, len(labels))
	for _, label := range labels {
		parts = append(parts, namedLink{label: label, url: links[label]})
	}
	return parts
}

func variantsForLanguage(key string, entry languageReference) []genericLanguage {
	start := entry.index
	end := start + entry.variants
	if start >= 0 && end <= len(genericLanguages) {
		variants := genericLanguages[start:end]
		valid := true
		for _, lang := range variants {
			if lang.Name != key {
				valid = false
				break
			}
		}
		if valid {
			return variants
		}
	}
	var variants []genericLanguage
	for _, lang := range genericLanguages {
		if lang.Name == key {
			variants = append(variants, lang)
		}
	}
	return variants
}

func sortLanguageNames(names []string) {
	sort.Slice(names, func(i int, j int) bool {
		leftFolded := strings.ToLower(names[i])
		rightFolded := strings.ToLower(names[j])
		if leftFolded == rightFolded {
			return names[i] < names[j]
		}
		return leftFolded < rightFolded
	})
}

func collectLanguageReports(selector *regexp.Regexp) []languageReport {
	names := make([]string, 0)
	for key, entry := range references {
		if strings.HasSuffix(key, "-like") {
			continue
		}
		if languageMatchesSelector(selector, key, entry.aka) {
			names = append(names, key)
		}
	}
	sortLanguageNames(names)

	reports := make([]languageReport, 0, len(names))
	for _, key := range names {
		entry := references[key]
		report := languageReport{
			name:        key,
			aliases:     parseAliases(entry.aka),
			description: entry.description,
			links:       referenceLinks(entry.links),
		}
		for _, lang := range variantsForLanguage(key, entry) {
			report.variants = append(report.variants, languageVariantReport{
				name:   lang.Var,
				traits: traitDescriptions(lang),
			})
		}
		reports = append(reports, report)
	}
	return reports
}

func slugifyName(s string) string {
	s = strings.ToLower(s)
	var b strings.Builder
	lastDash := false
	for _, r := range s {
		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
			b.WriteRune(r)
			lastDash = false
			continue
		}
		if !lastDash {
			b.WriteByte('-')
			lastDash = true
		}
	}
	out := strings.Trim(b.String(), "-")
	if out == "" {
		return "entry"
	}
	return out
}

func splitParagraphs(s string) []string {
	s = strings.TrimSpace(s)
	if s == "" {
		return nil
	}
	raw := blankLineRE.Split(s, -1)
	paragraphs := make([]string, 0, len(raw))
	for _, part := range raw {
		part = strings.TrimSpace(part)
		if part != "" {
			paragraphs = append(paragraphs, part)
		}
	}
	return paragraphs
}

func renderLanguageReportsPlain(reports []languageReport) string {
	var b strings.Builder
	for i, report := range reports {
		if i > 0 {
			b.WriteString("\n")
		}
		fmt.Fprintf(&b, "%s\n", report.name)
		if report.description != "" {
			fmt.Fprintf(&b, "%s\n", report.description)
		}
		if len(report.links) > 0 {
			b.WriteString("\n")
			for j, link := range report.links {
				if j > 0 {
					b.WriteString("\n\n")
				}
				fmt.Fprintf(&b, "%s::\n%s", link.label, link.url)
			}
			b.WriteString("\n")
		}
		for _, variant := range report.variants {
			b.WriteString("\n")
			if len(report.variants) > 1 {
				fmt.Fprintf(&b, "Variant: %s\n", variant.name)
			}
			for _, trait := range variant.traits {
				fmt.Fprintf(&b, "* %s\n", trait)
			}
		}
	}
	return b.String()
}

func renderLanguageReportsHTML(reports []languageReport, scope reportScope) string {
	var b strings.Builder
	mainClass := "language-report language-report-html language-report-selection"
	if scope == reportScopeAll {
		mainClass = "language-report language-report-html language-report-all"
	}
	b.WriteString("<!DOCTYPE html>\n<html>\n<head>\n<meta charset=\"utf-8\">\n<title>loccount language report</title>\n</head>\n<body>\n")
	fmt.Fprintf(&b, "<main class=\"%s\" data-report-format=\"html\" data-report-scope=\"%s\" data-entry-count=\"%d\">\n",
		mainClass,
		html.EscapeString(string(scope)),
		len(reports))
	for _, report := range reports {
		aliasAttr := html.EscapeString(strings.Join(report.aliases, "|"))
		fmt.Fprintf(&b, "<article id=\"language-%s\" class=\"language-entry\" data-name=\"%s\" data-aliases=\"%s\" data-variant-count=\"%d\" data-link-count=\"%d\">\n",
			html.EscapeString(slugifyName(report.name)),
			html.EscapeString(report.name),
			aliasAttr,
			len(report.variants),
			len(report.links))
		b.WriteString("<header class=\"language-header\">\n")
		fmt.Fprintf(&b, "<h1 class=\"language-name\">%s</h1>\n", html.EscapeString(report.name))
		b.WriteString("</header>\n")
		b.WriteString("<div class=\"language-description\">\n")
		for _, paragraph := range splitParagraphs(report.description) {
			fmt.Fprintf(&b, "<p class=\"language-description-paragraph\">%s</p>\n", html.EscapeString(paragraph))
		}
		b.WriteString("</div>\n")
		if len(report.links) > 0 {
			fmt.Fprintf(&b, "<dl class=\"language-links\" data-link-count=\"%d\">\n", len(report.links))
			for _, link := range report.links {
				fmt.Fprintf(&b, "<dt class=\"language-link-label\">%s</dt>\n", html.EscapeString(link.label))
				fmt.Fprintf(&b, "<dd class=\"language-link-value\"><a class=\"language-link\" href=\"%s\">%s</a></dd>\n",
					html.EscapeString(link.url),
					html.EscapeString(link.url))
			}
			b.WriteString("</dl>\n")
		}
		b.WriteString("<div class=\"language-variants\">\n")
		for _, variant := range report.variants {
			fmt.Fprintf(&b, "<section class=\"language-variant\" data-variant-name=\"%s\" data-trait-count=\"%d\">\n",
				html.EscapeString(variant.name),
				len(variant.traits))
			if len(report.variants) > 1 {
				fmt.Fprintf(&b, "<h2 class=\"language-variant-name\">Variant: %s</h2>\n", html.EscapeString(variant.name))
			}
			fmt.Fprintf(&b, "<ul class=\"language-traits\" data-trait-count=\"%d\">\n", len(variant.traits))
			for _, trait := range variant.traits {
				fmt.Fprintf(&b, "<li class=\"language-trait\">%s</li>\n", html.EscapeString(trait))
			}
			b.WriteString("</ul>\n</section>\n")
		}
		b.WriteString("</div>\n")
		b.WriteString("</article>\n")
	}
	b.WriteString("</main>\n</body>\n</html>\n")
	return b.String()
}

func renderLanguageReports(reports []languageReport, format selectorReportFormat, scope reportScope) string {
	if format == selectorReportHTML {
		return renderLanguageReportsHTML(reports, scope)
	}
	return renderLanguageReportsPlain(reports)
}

func describeLanguages(selector *regexp.Regexp, format selectorReportFormat) {
	fmt.Print(renderLanguageReports(collectLanguageReports(selector), format, reportScopeSelection))
}

func describeAllLanguages(format selectorReportFormat) {
	fmt.Print(renderLanguageReports(collectLanguageReports(nil), format, reportScopeAll))
}

// Main sequence

type sortable []countRecord

func (a sortable) Len() int           { return len(a) }
func (a sortable) Swap(i int, j int)  { a[i], a[j] = a[j], a[i] }
func (a sortable) Less(i, j int) bool { return -a[i].slinecount < -a[j].slinecount }

var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")

func main() {
	var cocomo bool
	var extensions bool
	var htmlreport bool
	var individual bool
	var json bool
	var keepwalk bool
	var llist bool
	var noGitmodules bool
	var noNeighborInference bool
	var nodocs bool
	var slist bool
	var showversion bool
	var unclassified bool
	var webdump bool
	flag.BoolVar(&cocomo, "c", false,
		"report Cocomo-model estimation")
	flag.IntVar(&debug, "d", 0,
		"set debug level")
	flag.BoolVar(&extensions, "e", false,
		"list extensions associated with each language and exit")
	flag.BoolVar(&listignores, "g", false,
		"list generated files")
	flag.BoolVar(&htmlreport, "H", false,
		"with -t, emit HTML instead of plain text")
	flag.BoolVar(&individual, "i", false,
		"list counts and types for individual files")
	flag.BoolVar(&json, "j", false,
		"dump statistics in JSON format")
	flag.BoolVar(&keepwalk, "k", false,
		"keep regular directory traversal even inside recognized VCS working copies")
	flag.BoolVar(&llist, "l", false,
		"list languages that yield LLOC and exit")
	flag.BoolVar(&noNeighborInference, "N", false,
		"do not infer ambiguous file types from other files in the tree")
	flag.BoolVar(&noGitmodules, "no-gitmodules", false,
		"disable .gitmodules handling during regular directory traversal")
	flag.BoolVar(&nodocs, "n", false,
		"do not tally documentation")
	flag.BoolVar(&slist, "s", false,
		"list languages that yield SLOC and exit")
	selector := flag.String("t", "",
		"describe matching languages and exit (regexp)")
	flag.BoolVar(&unclassified, "u", false,
		"list unclassified files")
	flag.BoolVar(&transdump, "v", false,
		"Dump transitions in output.")
	flag.BoolVar(&showversion, "V", false,
		"report version and exit")
	flag.BoolVar(&webdump, "W", false,
		"emit an HTML page describing all languages and exit")
	excludePtr := flag.String("x", "",
		"paths and directories to exclude (regexp)")
	flag.Parse()

	if htmlreport && len(*selector) == 0 {
		log.Fatal("loccount: -H requires -t")
	}
	if webdump && len(*selector) > 0 {
		log.Fatal("loccount: -W conflicts with -t")
	}
	if webdump && htmlreport {
		log.Fatal("loccount: -W conflicts with -H")
	}

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}
	if showversion {
		fmt.Printf("loccount %s\n", version)
		return
	} else if slist {
		ll, duplicates := listLanguages(false, nodocs)
		if !individual {
			fmt.Printf("%d: %s\n", len(ll), ll)
		} else {
			for _, lang := range ll {
				fmt.Printf("%s\n", lang)
			}
		}
		if duplicates {
			os.Exit(1)
		}
		return
	} else if llist {
		ll, _ := listLanguages(true, nodocs)
		if !individual {
			fmt.Printf("%d: %s\n", len(ll), ll)
		} else {
			for _, lang := range ll {
				fmt.Printf("%s\n", lang)
			}
		}
		return
	} else if extensions {
		listExtensions()
		return
	} else if webdump {
		describeAllLanguages(selectorReportHTML)
		return
	} else if len(*selector) > 0 {
		selectorRE, err := regexp.Compile(*selector)
		if err != nil {
			log.Fatalf("loccount: invalid -t regexp %q: %v", *selector, err)
		}
		format := selectorReportPlain
		if htmlreport {
			format = selectorReportHTML
		}
		describeLanguages(selectorRE, format)
		return
	}

	individual = individual || unclassified

	// For maximum performance, make the pipeline be as deep as the
	// number of processor we have available, that way the machine will
	// be running full-out exactly when it's filled and no sooner.
	// This makes order of output nondeterministic, which is why
	// we sometimes want to disable it.
	var chandepth int
	var workerCount int
	if individual || unclassified {
		chandepth = 0
		workerCount = 1
	} else {
		chandepth = runtime.NumCPU()
		workerCount = runtime.NumCPU()
	}
	pipeline = make(chan SourceStat, chandepth)

	if len(*excludePtr) > 0 {
		exclusions = regexp.MustCompile(*excludePtr)
	}
	roots := flag.Args()

	go func() {
		walkRoots(roots, keepwalk, noGitmodules, workerCount)
		close(pipeline)
	}()

	var totals countRecord
	counts := map[string]countRecord{}
	results := make([]SourceStat, 0)

	// Mainline resumes
	for {
		st, more := <-pipeline
		if !more {
			break
		}
		if debug > 0 {
			fmt.Printf("from pipeline: %s %d %s\n",
				st.Path, st.SLOC, st.Language)
		}
		results = append(results, st)
	}

	inferUniqueContentMatchedTypes(results, !noNeighborInference)

	if individual {
		for _, st := range results {
			if !unclassified && st.SLOC > 0 {
				fmt.Printf("%s:%s:%d:%d\n",
					st.Path, st.reportLanguage(), st.SLOC, st.LLOC)
			} else if unclassified && st.SLOC == 0 {
				// Not a recognized source type,
				// nor anything we know to discard
				fmt.Println(st.Path)
			}
		}
		return
	}

	for _, st := range results {
		if st.SLOC > 0 && !(nodocs && st.markup) {
			key := st.reportLanguage()
			var tmp = counts[key]
			tmp.language = st.Language
			tmp.cgi = st.cgi
			tmp.slinecount += st.SLOC
			tmp.llinecount += st.LLOC
			tmp.filecount++
			counts[key] = tmp
			totals.slinecount += st.SLOC
			totals.llinecount += st.LLOC
			totals.filecount++
		}
	}

	// C headers may get reassigned based on what other languages
	// are present in the tree
	reassignCHeaderCounts(counts)

	var summary sortable
	totals.language = "all"
	if totals.filecount > 1 {
		summary = append(summary, totals)
	}
	for _, v := range counts {
		summary = append(summary, v)
	}

	sort.Sort(summary)
	for i := range summary {
		r := summary[i]
		if json {
			fmt.Printf("{\"language\":%q, \"sloc\":%d, \"lloc\":%d, \"filecount\":%d}\n",
				r.reportLanguage(),
				r.slinecount,
				r.llinecount,
				r.filecount)
		} else {
			fmt.Printf("%-20s SLOC=%-7d (%2.2f%%)\tLLOC=%-7d in %d files\n",
				r.reportLanguage(),
				r.slinecount,
				float64(r.slinecount)*100.0/float64(totals.slinecount),
				r.llinecount,
				r.filecount)
		}
	}

	if cocomo {
		reportCocomo(totals.slinecount, cocomo81)
		reportCocomo(totals.llinecount, cocomo2000)
	}
}

// end
