carbon-tokenization

Running

App Files Files Community

carbon-tokenization / backend /tests /highlight-autodetect.test.ts

tfrere HF Staff

Cursor

fix(highlight): auto-detect language for code blocks without explicit lang

bf2abd0 5 months ago

Raw History Blame Contribute Delete

2 kB

/**
 * Auto-detection of syntax highlighting for language-less code blocks.
 *
 * Code blocks authored without an explicit language must still be highlighted
 * (in both the editor and the published output). These tests cover the
 * publisher path end-to-end plus the shared detector in isolation.
 */
import { describe, it, expect } from "vitest";
import { renderArticleHTML, type PublishMeta } from "../src/publisher/html-renderer.js";
import type { PublishCSS } from "../src/publisher/index.js";
import { detectShikiLang } from "../src/shared/detect-lang.js";

const EMPTY_CSS: PublishCSS = {
  variables: "",
  reset: "",
  base: "",
  layout: "",
  print: "",
  editorTokens: "",
  article: "",
  components: "",
  publisher: "",
};

const META: PublishMeta = {
  title: "Code Article",
  description: "code highlighting",
  authors: [{ name: "Alice", affiliationIndices: [1], affiliationNames: ["MIT"] }],
  affiliations: [{ name: "MIT" }],
  date: "2025-01-01",
};

const PY = `import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
def load(name):
    return AutoModelForCausalLM.from_pretrained(name)`;

describe("detectShikiLang", () => {
  it("detects python from a typical snippet", () => {
    expect(detectShikiLang(PY)).toBe("python");
  });

  it("returns empty for trivially short input", () => {
    expect(detectShikiLang("x")).toBe("");
  });
});

describe("publisher highlights language-less code blocks", () => {
  it("colors a code block that has no language attribute", async () => {
    const json = {
      type: "doc",
      content: [
        // Note: no `attrs.language` - mirrors how these blocks are authored.
        { type: "codeBlock", content: [{ type: "text", text: PY }] },
      ],
    };
    const html = await renderArticleHTML(json as any, META, EMPTY_CSS);

    // Detected language label and per-token color vars must be present.
    expect(html).toContain('data-lang="python"');
    expect(html).toContain("--shiki-light:");
  });
});

	/**
	* Auto-detection of syntax highlighting for language-less code blocks.
	*
	* Code blocks authored without an explicit language must still be highlighted
	* (in both the editor and the published output). These tests cover the
	* publisher path end-to-end plus the shared detector in isolation.
	*/
	import { describe, it, expect } from "vitest";
	import { renderArticleHTML, type PublishMeta } from "../src/publisher/html-renderer.js";
	import type { PublishCSS } from "../src/publisher/index.js";
	import { detectShikiLang } from "../src/shared/detect-lang.js";

	const EMPTY_CSS: PublishCSS = {
	variables: "",
	reset: "",
	base: "",
	layout: "",
	print: "",
	editorTokens: "",
	article: "",
	components: "",
	publisher: "",
	};

	const META: PublishMeta = {
	title: "Code Article",
	description: "code highlighting",
	authors: [{ name: "Alice", affiliationIndices: [1], affiliationNames: ["MIT"] }],
	affiliations: [{ name: "MIT" }],
	date: "2025-01-01",
	};

	const PY = `import torch
	from transformers import AutoModelForCausalLM, AutoTokenizer

	def load(name):
	return AutoModelForCausalLM.from_pretrained(name)`;

	describe("detectShikiLang", () => {
	it("detects python from a typical snippet", () => {
	expect(detectShikiLang(PY)).toBe("python");
	});

	it("returns empty for trivially short input", () => {
	expect(detectShikiLang("x")).toBe("");
	});
	});

	describe("publisher highlights language-less code blocks", () => {
	it("colors a code block that has no language attribute", async () => {
	const json = {
	type: "doc",
	content: [
	// Note: no `attrs.language` - mirrors how these blocks are authored.
	{ type: "codeBlock", content: [{ type: "text", text: PY }] },
	],
	};
	const html = await renderArticleHTML(json as any, META, EMPTY_CSS);

	// Detected language label and per-token color vars must be present.
	expect(html).toContain('data-lang="python"');
	expect(html).toContain("--shiki-light:");
	});
	});