typespec_audit.exs

#!/usr/bin/env elixir

# run with: mix run scripts/typespec_audit.exs --namespace=AppName   

defmodule TypespecAudit do
  @moduledoc """
  Script to analyze typespec coverage across an Elixir codebase.

  This script:
  1. Finds all Elixir modules in the codebase
  2. Analyzes each module for typespec coverage
  3. Generates a report of modules with missing typespecs
  4. Provides metrics on overall typespec health

  Usage:
    mix run scripts/typespec_audit.exs [--namespace=AppName] [--depth=2] [--quiet]
  """

  def run do
    # Parse command line arguments
    args = System.argv()
    options = parse_args(args)

    # Get application name from mix project or command line args
    app_name = get_app_name(options)
    namespace_prefix = options[:namespace] || app_name
    grouping_depth = options[:depth] || 2
    quiet_mode = options[:quiet] || false

    # Hide compiler warnings if quiet mode is enabled
    if quiet_mode do
      # Simply run compile first to make sure all modules are compiled
      # before we do the analysis. This way we won't need to compile again.
      {_, 0} = System.cmd("mix", ["compile"], stderr_to_stdout: true)
    end

    IO.puts("\n==== #{namespace_prefix} Typespec Coverage Audit ====\n")

    # Get all modules (use both file-based discovery and loaded modules)
    all_modules = discover_modules(namespace_prefix, quiet_mode)
    components = discover_components(all_modules, grouping_depth)

    IO.puts("Found #{length(all_modules)} modules to analyze\n")

    # Analyze typespec coverage by component
    component_reports =
      components
      |> Enum.map(fn {component_name, component_modules} ->
        IO.puts("Component #{component_name} has #{length(component_modules)} modules")

        # Analyze each module in the component
        module_reports =
          component_modules
          |> Enum.map(&analyze_module/1)
          |> Enum.reject(&is_nil/1)
          |> Enum.sort_by(fn %{coverage_pct: pct} -> pct end)

        # Calculate component-level metrics
        total_functions = Enum.reduce(module_reports, 0, & &1.total_functions + &2)
        total_with_specs = Enum.reduce(module_reports, 0, & &1.with_specs + &2)
        component_coverage = if total_functions > 0, do: total_with_specs / total_functions * 100, else: 0.0

        %{
          name: component_name,
          modules: module_reports,
          module_count: length(module_reports),
          total_functions: total_functions,
          with_specs: total_with_specs,
          coverage_pct: (if component_coverage > 0, do: Float.round(component_coverage, 1), else: 0.0)
        }
      end)
      |> Enum.sort_by(fn %{coverage_pct: pct} -> pct end)

    # Print component reports
    Enum.each(component_reports, &print_component_report/1)

    # Calculate overall metrics
    total_modules = Enum.reduce(component_reports, 0, & &1.module_count + &2)
    total_functions = Enum.reduce(component_reports, 0, & &1.total_functions + &2)
    total_with_specs = Enum.reduce(component_reports, 0, & &1.with_specs + &2)
    overall_coverage = if total_functions > 0, do: total_with_specs / total_functions * 100, else: 0.0

    # Print overall summary
    IO.puts("\n==== Overall Coverage Summary ====")
    IO.puts("Total Modules: #{total_modules}")
    IO.puts("Total Public Functions: #{total_functions}")
    IO.puts("Functions with @spec: #{total_with_specs}")
    IO.puts("Overall Coverage: #{(if overall_coverage > 0, do: Float.round(overall_coverage, 1), else: 0.0)}%")

    # Generate recommendations
    generate_recommendations(component_reports)
  end

  defp parse_args(args) do
    {options, _, _} =
      OptionParser.parse(args,
        strict: [
          namespace: :string,
          depth: :integer,
          quiet: :boolean
        ]
      )

    options
    |> Enum.into(%{})
  end

  defp get_app_name(options) do
    cond do
      # Use explicitly provided namespace
      options[:namespace] ->
        options[:namespace]

      # Try getting from mix project
      Code.ensure_loaded?(Mix.Project) ->
        Mix.Project.config()[:app] |> to_string() |> Macro.camelize()

      # Default fallback
      true ->
        IO.puts("⚠️  Warning: Could not determine application name. Use --namespace=YourApp to specify.")
        "Elixir"
    end
  end

    defp discover_modules(namespace_prefix, _quiet_mode) do
    # Find all modules in the codebase that match our namespace
    IO.puts("Discovering modules...")

    # Combine modules found from loaded modules and file paths
    loaded_modules = discover_from_loaded_modules(namespace_prefix)
    file_modules = discover_from_file_paths(namespace_prefix)

    # Combine and deduplicate
    all_modules = (loaded_modules ++ file_modules) |> Enum.uniq()

    IO.puts("Found #{length(all_modules)} modules")
    all_modules
  end

  defp discover_from_loaded_modules(namespace_prefix) do
    IO.puts("Finding already loaded modules...")

    loaded_modules =
      for {module, _} <- :code.all_loaded(),
          module_name = Atom.to_string(module),
          String.starts_with?(module_name, "Elixir.#{namespace_prefix}.") do
        module
      end

    IO.puts("Found #{length(loaded_modules)} loaded modules")
    loaded_modules
  end

  defp discover_from_file_paths(namespace_prefix) do
    IO.puts("Scanning source files...")
    source_files = find_elixir_files(["lib/"])
    IO.puts("Found #{length(source_files)} source files")

    # Extract modules from file content
    modules =
      source_files
      |> Enum.flat_map(fn file ->
        extract_modules_from_file(file, namespace_prefix)
      end)
      |> Enum.uniq()

    IO.puts("Extracted #{length(modules)} modules from source files")
    modules
  end

  defp extract_modules_from_file(file_path, namespace_prefix) do
    try do
      case File.read(file_path) do
        {:ok, content} ->
          # Look for all defmodule declarations in the file content
          # This regex will capture all module names in the file that match the namespace
          regex = ~r/defmodule\s+(#{namespace_prefix}[.\w]+)\s+do/

          Regex.scan(regex, content)
          |> Enum.map(fn [_, module_name] ->
            # Convert to atom to get the module
            String.to_atom("Elixir.#{module_name}")
          end)

        _ -> []
      end
    rescue
      _ -> []
    end
  end

  defp find_elixir_files(directories) do
    directories
    |> Enum.flat_map(fn dir ->
      case File.ls(dir) do
        {:ok, files} ->
          Enum.flat_map(files, fn file ->
            path = Path.join(dir, file)
            cond do
              # If it's a directory, recursively search it
              File.dir?(path) ->
                find_elixir_files([path])
              # If it's a .ex file, include it
              String.ends_with?(file, ".ex") ->
                [path]
              # Otherwise, skip it
              true ->
                []
            end
          end)
        _ -> []
      end
    end)
  end

  defp discover_components(modules, grouping_depth) do
    # Group modules by components based on namespace structure
    modules
    |> Enum.reduce(%{}, fn module, acc ->
      # Extract component name from module name
      module_string = Atom.to_string(module)
      # Strip the "Elixir." prefix
      module_string = String.replace_prefix(module_string, "Elixir.", "")

      # Split into namespace parts
      parts = String.split(module_string, ".")

      # Create component name based on the specified grouping depth
      # Get parts up to grouping_depth or all parts if fewer
      component_parts = Enum.take(parts, min(grouping_depth, length(parts)))

      # For modules with just top-level namespace, use "Core"
      component_name = if length(component_parts) <= 1, do: "Core", else: Enum.join(component_parts, ".")

      # Add the module to its component group
      Map.update(acc, component_name, [module], fn modules -> [module | modules] end)
    end)
    |> Map.to_list()
  end

  defp analyze_module(module) do
    IO.puts("Analyzing module: #{inspect(module)}")

    # Make sure the module is loaded
    if ensure_module_loaded(module) do
      # Get all public functions using reflection
      public_functions =
        try do
          functions = module.__info__(:functions)
          IO.puts("  Found #{length(functions)} raw functions in #{inspect(module)}")

          filtered_functions = functions
          |> Enum.reject(fn {name, _arity} ->
            # Filter out typical non-public or automatically generated functions
            name in [:__struct__, :module_info, :behaviour_info, :__impl__]
          end)

          IO.puts("  After filtering, #{length(filtered_functions)} public functions remain")
          filtered_functions
        rescue
          e ->
            IO.puts("❌ Error getting functions for #{inspect(module)}: #{inspect(e)}")
            []
        end

      # Get all functions with typespecs
      specs =
        try do
          case Code.Typespec.fetch_specs(module) do
            {:ok, specs} ->
              IO.puts("  Found #{length(specs)} specs in #{inspect(module)}")
              specs
            _ ->
              IO.puts("  No specs found in #{inspect(module)}")
              []
          end
        rescue
          e ->
            IO.puts("❌ Error fetching specs for #{inspect(module)}: #{inspect(e)}")
            []
        end

      functions_with_specs = Enum.map(specs, fn {{name, arity}, _} -> {name, arity} end)
      IO.puts("  Functions with specs: #{length(functions_with_specs)}")

      # Calculate coverage metrics
      total_functions = length(public_functions)
      with_specs = length(functions_with_specs)
      coverage_pct = if total_functions > 0, do: with_specs / total_functions * 100, else: 0.0

      # Find functions missing specs
      missing_specs = public_functions -- functions_with_specs

      # Check for custom types
      types =
        try do
          case Code.Typespec.fetch_types(module) do
            {:ok, types} ->
              IO.puts("  Found #{length(types)} type definitions in #{inspect(module)}")
              types
            _ ->
              IO.puts("  No type definitions found in #{inspect(module)}")
              []
          end
        rescue
          e ->
            IO.puts("❌ Error fetching types for #{inspect(module)}: #{inspect(e)}")
            []
        end

      # Handle different type formats safely
      custom_types =
        try do
          Enum.count(types, fn
            {kind, _, _} when kind in [:type, :opaque] -> true
            _ -> false
          end)
        rescue
          _ ->
            IO.puts("  ⚠️ Error counting custom types, using count 0")
            0
        end

      # Only return the report if we have functions to analyze
      if total_functions > 0 do
        %{
          module: module,
          total_functions: total_functions,
          with_specs: with_specs,
          missing_specs: missing_specs,
          coverage_pct: (if coverage_pct > 0, do: Float.round(coverage_pct, 1), else: 0.0),
          custom_types: custom_types
        }
      else
        IO.puts("  Module has no public functions to analyze, skipping.")
        nil
      end
    else
      IO.puts("  Module could not be loaded, skipping.")
      nil
    end
  end

  defp ensure_module_loaded(module) do
    try do
      # Try to load and compile the module
      case Code.ensure_loaded(module) do
        {:module, _} -> true
        _ -> false
      end
    rescue
      _ -> false
    catch
      _, _ -> false
    end
  end

  defp print_component_report(%{name: name, modules: modules, coverage_pct: coverage_pct}) do
    IO.puts("\n=== Component: #{name} (#{coverage_pct}% covered) ===")

    if Enum.empty?(modules) do
      IO.puts("No modules found in this component.")
      nil
    else
      if coverage_pct < 50 do
        IO.puts("⚠️  LOW COVERAGE - Priority for improvement")
      end

      Enum.each(modules, fn module_report ->
        %{
          module: module,
          total_functions: total,
          with_specs: with_specs,
          coverage_pct: module_coverage,
          missing_specs: missing,
          custom_types: custom_types
        } = module_report

        status_indicator = cond do
          module_coverage >= 90 -> "✅"
          module_coverage >= 70 -> "🟡"
          module_coverage >= 30 -> "🟠"
          true -> "🔴"
        end

        IO.puts("#{status_indicator} #{inspect(module)}: #{with_specs}/#{total} functions typed (#{module_coverage}%)")

        if custom_types > 0 do
          IO.puts("   └─ Defines #{custom_types} custom type(s)")
        end

        if length(missing) > 0 && length(missing) <= 5 do
          missing_formatted =
            Enum.map(missing, fn {name, arity} -> "#{name}/#{arity}" end)
            |> Enum.join(", ")

          IO.puts("   └─ Missing specs: #{missing_formatted}")
        else
          if length(missing) > 5 do
            IO.puts("   └─ Missing specs on #{length(missing)} functions")
          end
        end
      end)
    end
  end

  defp generate_recommendations(component_reports) do
    IO.puts("\n==== Recommendations ====")

    # Find components with lowest coverage
    low_coverage_components =
      component_reports
      |> Enum.filter(fn %{coverage_pct: pct} -> pct < 70 end)
      |> Enum.sort_by(fn %{coverage_pct: pct} -> pct end)
      |> Enum.take(3)

    if length(low_coverage_components) > 0 do
      IO.puts("\nPriority Components for Improvement:")
      Enum.each(low_coverage_components, fn %{name: name, coverage_pct: pct} ->
        IO.puts("- #{name} (#{pct}% coverage)")
      end)
    end

    # Find modules with highest function count but low coverage
    high_impact_modules =
      component_reports
      |> Enum.flat_map(fn %{modules: modules} -> modules end)
      |> Enum.filter(fn %{total_functions: total, coverage_pct: pct} ->
        total > 5 && pct < 70
      end)
      |> Enum.sort_by(fn %{total_functions: total} -> -total end)
      |> Enum.take(5)

    if length(high_impact_modules) > 0 do
      IO.puts("\nHigh-Impact Modules for Improvement:")
      Enum.each(high_impact_modules, fn %{module: module, total_functions: total, coverage_pct: pct} ->
        IO.puts("- #{inspect(module)}: #{total} functions, only #{pct}% covered")
      end)
    end

    # Find potential type module candidates (modules with many custom types)
    type_module_candidates =
      component_reports
      |> Enum.flat_map(fn %{modules: modules} -> modules end)
      |> Enum.filter(fn %{custom_types: types} -> types > 2 end)
      |> Enum.sort_by(fn %{custom_types: types} -> -types end)
      |> Enum.take(3)

    if length(type_module_candidates) > 0 do
      IO.puts("\nPotential Type Module Candidates:")
      Enum.each(type_module_candidates, fn %{module: module, custom_types: types} ->
        IO.puts("- #{inspect(module)}: defines #{types} custom types")
      end)
    end
  end
end

# Run the audit when the script is executed
TypespecAudit.run()