Move init templates to a separate package

.chronus/changes/init-template-pkg-2025-3-4-15-48-47.md

@@ -0,0 +1,9 @@
+---
+# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking
+changeKind: internal
+packages:
+  - "@typespec/compiler"
+  - "@typespec/init-templates"
+---
+
+Move init templates to a separate package

packages/compiler/package.json

@@ -77,8 +77,7 @@
   ],
   "scripts": {
     "clean": "rimraf ./dist ./temp",
-    "build:init-templates-index": "tsx ./.scripts/build-init-templates.ts",
-    "build": "pnpm gen-manifest && pnpm build:init-templates-index && pnpm compile && pnpm generate-tmlanguage",
+    "build": "pnpm gen-manifest && pnpm compile && pnpm generate-tmlanguage",
     "api-extractor": "api-extractor run --local --verbose",
     "compile": "tsc -p .",
     "watch": "tsc -p . --watch",

packages/compiler/src/cli/common.ts

@@ -0,0 +1,15 @@
+import envPaths from "env-paths";
+import { joinPaths } from "../core/path-utils.js";
+
+const paths = envPaths("typespec", { suffix: "" });
+
+export const KnownDirectories = {
+  /**
+   * The directory where the package manager are installed.
+   */
+  packageManager: joinPaths(paths.cache, "pm"),
+  /**
+   * The directory where the init template package is installed.
+   */
+  initTemplates: joinPaths(paths.cache, "init-templates"),
+} as const;

packages/compiler/src/init/core-templates.ts

@@ -1,22 +1,43 @@
-import { CompilerPackageRoot } from "../core/node-host.js";
-import { resolvePath } from "../core/path-utils.js";
-import type { SystemHost } from "../core/types.js";
+import { mkdir, readFile, writeFile } from "fs/promises";
+import { KnownDirectories } from "../cli/common.js";
+import { joinPaths } from "../core/path-utils.js";
+import { downloadPackageVersion } from "../package-manger/npm-registry-utils.js";
 
-export const templatesDir = resolvePath(CompilerPackageRoot, "templates");
 export interface LoadedCoreTemplates {
   readonly baseUri: string;
   readonly templates: Record<string, any>;
 }
 
-let typeSpecCoreTemplates: LoadedCoreTemplates | undefined;
-export async function getTypeSpecCoreTemplates(host: SystemHost): Promise<LoadedCoreTemplates> {
-  if (typeSpecCoreTemplates === undefined) {
-    const file = await host.readFile(resolvePath(templatesDir, "scaffolding.json"));
-    const content = JSON.parse(file.text);
-    typeSpecCoreTemplates = {
-      baseUri: templatesDir,
-      templates: content,
-    };
+const LAST_CHECK_FILE_NAME = ".last-check.json";
+const CHECK_TIMEOUT = 86400_000; // 24 hours in milliseconds
+
+export async function getTypeSpecCoreTemplates(): Promise<LoadedCoreTemplates> {
+  const lastCheck = await readLastCheckFile();
+  if (lastCheck === undefined || new Date().getTime() > lastCheck?.getTime() + CHECK_TIMEOUT) {
+    await mkdir(KnownDirectories.initTemplates, { recursive: true });
+    await downloadPackageVersion("@typespec/compiler", "latest", KnownDirectories.initTemplates);
+    await saveLastCheckFile();
   }
-  return typeSpecCoreTemplates;
+
+  return (await import(`${KnownDirectories.initTemplates}/src/index.js`)).default;
+}
+
+async function readLastCheckFile(): Promise<Date | undefined> {
+  try {
+    const lastCheck = await readFile(
+      joinPaths(KnownDirectories.initTemplates, LAST_CHECK_FILE_NAME),
+      "utf8",
+    );
+    return new Date(JSON.parse(lastCheck).time);
+  } catch (e) {
+    return undefined;
+  }
+}
+
+async function saveLastCheckFile() {
+  await writeFile(
+    joinPaths(KnownDirectories.initTemplates, LAST_CHECK_FILE_NAME),
+    JSON.stringify({ time: new Date().toISOString() }, null, 2),
+    "utf8",
+  );
 }

packages/compiler/src/init/init.ts

@@ -51,7 +51,7 @@ export async function initTypeSpecProjectWorker(
 
   // Download template configuration and prompt user to select a template
   // No validation is done until one has been selected
-  const typeSpecCoreTemplates = await getTypeSpecCoreTemplates(host);
+  const typeSpecCoreTemplates = await getTypeSpecCoreTemplates();
   const result =
     options.templatesUrl === undefined
       ? (typeSpecCoreTemplates as LoadedTemplate)

packages/compiler/src/package-manger/npm-registry-utils.ts

@@ -93,7 +93,11 @@ export async function fetchPackageManifest(
 ): Promise<NpmManifest> {
   const url = `${registry}/${packageName}/${version}`;
   const res = await fetch(url);
-  return await res.json();
+  if (res.ok) {
+    return await res.json();
+  } else {
+    throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`);
+  }
 }
 
 export function fetchLatestPackageManifest(packageName: string): Promise<NpmManifest> {

packages/compiler/src/server/serverlib.ts

@@ -313,7 +313,7 @@ export function createServer(host: ServerHost): Server {
 
   async function getInitProjectContext(): Promise<InitProjectContext> {
     return {
-      coreInitTemplates: await getTypeSpecCoreTemplates(host.compilerHost),
+      coreInitTemplates: await getTypeSpecCoreTemplates(),
     };
   }
 

packages/compiler/test/e2e/init-templates.e2e.ts

@@ -63,7 +63,7 @@ describe("Init templates e2e tests", () => {
   });
 
   async function scaffoldTemplateTo(name: string, targetFolder: string) {
-    const typeSpecCoreTemplates = await getTypeSpecCoreTemplates(NodeHost);
+    const typeSpecCoreTemplates = await getTypeSpecCoreTemplates();
     const template = typeSpecCoreTemplates.templates[name];
     ok(template, `Template '${name}' not found`);
     await scaffoldNewProject(

packages/init-templates/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@typespec/init-templates",
+  "version": "0.68.0",
+  "author": "Microsoft Corporation",
+  "description": "TypeSpec init templates",
+  "homepage": "https://typespec.io",
+  "readme": "https://github.com/microsoft/typespec/blob/main/README.md",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/microsoft/typespec.git"
+  },
+  "bugs": {
+    "url": "https://github.com/microsoft/typespec/issues"
+  },
+  "keywords": [
+    "typespec"
+  ],
+  "type": "module",
+  "exports": {
+    ".": {
+      "import": "./src/index.js"
+    }
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "scripts": {
+    "clean": "rimraf ./dist ./temp",
+    "build": "tsx ./scripts/build.ts",
+    "test": "pnpm test:e2e",
+    "test:e2e": "vitest run --config ./vitest.config.e2e.ts",
+    "lint": "eslint . --ext .ts --max-warnings=0",
+    "lint:fix": "eslint . --fix --ext .ts"
+  },
+  "files": [
+    "lib/*.tsp",
+    "dist/**",
+    "!dist/test/**"
+  ],
+  "//": "IMPORTANT THIS PACKAGE CANNOT HAVE A DEPENDENCY",
+  "devDependencies": {
+    "@types/node": "~22.13.11",
+    "@typespec/compiler": "workspace:^",
+    "@vitest/coverage-v8": "^3.0.9",
+    "@vitest/ui": "^3.0.9",
+    "c8": "^10.1.3",
+    "pathe": "^2.0.3",
+    "rimraf": "~6.0.1",
+    "typescript": "~5.8.2",
+    "vitest": "^3.0.9"
+  }
+}

packages/compiler/.scripts/build-init-templates.ts → packages/init-templates/scripts/build.ts

@@ -1,12 +1,10 @@
-import { mkdir, readFile, writeFile } from "fs/promises";
-import { resolve } from "path/posix";
-import type { InitTemplate } from "../src/init/init-template.js";
+import { MANIFEST } from "@typespec/compiler";
+import { mkdir, writeFile } from "fs/promises";
+import { resolve } from "pathe";
+// @ts-ignore
+import type { InitTemplate } from "../../compiler/src/init/init-template.js";
 import { localDir, packageRoot } from "./helpers.js";
-
-const pkgJson = JSON.parse(
-  (await readFile(resolve(packageRoot, "package.json"))).toString("utf-8"),
-);
-const minCompilerVersion = pkgJson.version;
+const minCompilerVersion = MANIFEST.version;
 
 const builtInTemplates: Record<string, InitTemplate> = {
   rest: {

packages/compiler/.scripts/helpers.ts → packages/init-templates/scripts/helpers.ts

@@ -1,6 +1,6 @@
 import { readdir } from "fs/promises";
 import { dirname } from "path";
-import { join, resolve } from "path/posix";
+import { join, resolve } from "pathe";
 import { fileURLToPath } from "url";
 
 export const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");

packages/init-templates/src/index.d.ts

@@ -0,0 +1,6 @@
+declare const _default: {
+  readonly baseUri: string;
+  readonly templates: Record<string, any>;
+};
+
+export default _default;

packages/init-templates/src/index.js

@@ -0,0 +1,10 @@
+// @ts-check
+import { resolve } from "path";
+import scaffoldingJson from "../templates/scaffolding.json" with { type: "json" };
+
+export const templatesDir = resolve(import.meta.dirname, "../templates").replace(/\\/g, "/");
+
+export default {
+  baseUri: templatesDir,
+  templates: scaffoldingJson,
+};

packages/init-templates/test/init-templates.e2e.ts

@@ -0,0 +1,146 @@
+// Enable calling @typespec/compiler/internals for the init logic
+(globalThis as any).enableCompilerInternalsExport = true;
+import { NodeHost } from "@typespec/compiler";
+import typeSpecCoreTemplates from "@typespec/init-templates";
+import { ok } from "assert";
+import { type SpawnOptions, spawn } from "child_process";
+import { rm } from "fs/promises";
+import { resolve } from "pathe";
+import { beforeAll, describe, it } from "vitest";
+
+const projectRoot = resolve(import.meta.dirname, "..");
+const testTempRoot = resolve(projectRoot, "temp/scaffolded-template-tests");
+const snapshotFolder = resolve(projectRoot, "templates/__snapshots__");
+
+async function execAsync(
+  command: string,
+  args: string[] = [],
+  options: SpawnOptions = {},
+): Promise<{ exitCode: number; stdio: string; stdout: string; stderr: string; proc: any }> {
+  const child = spawn(command, args, options);
+
+  return new Promise((resolve, reject) => {
+    child.on("error", (error) => {
+      reject(error);
+    });
+    const stdio: Buffer[] = [];
+    const stdout: Buffer[] = [];
+    const stderr: Buffer[] = [];
+    child.stdout?.on("data", (data) => {
+      stdout.push(data);
+      stdio.push(data);
+    });
+    child.stderr?.on("data", (data) => {
+      stderr.push(data);
+      stdio.push(data);
+    });
+
+    child.on("exit", (exitCode) => {
+      resolve({
+        exitCode: exitCode ?? -1,
+        stdio: Buffer.concat(stdio).toString(),
+        stdout: Buffer.concat(stdout).toString(),
+        stderr: Buffer.concat(stderr).toString(),
+        proc: child,
+      });
+    });
+  });
+}
+
+interface ScaffoldedTemplateFixture {
+  /** Directory where the template was created. */
+  readonly directory: string;
+  readonly checkCommand: (
+    command: string,
+    args?: string[],
+    options?: SpawnOptions,
+  ) => Promise<void>;
+}
+
+describe("Init templates e2e tests", () => {
+  beforeAll(async () => {
+    await rm(testTempRoot, { recursive: true, force: true });
+  });
+
+  async function scaffoldTemplateTo(name: string, targetFolder: string) {
+    const { makeScaffoldingConfig, scaffoldNewProject } = await import(
+      "@typespec/compiler/internals"
+    );
+
+    const template = typeSpecCoreTemplates.templates[name];
+    ok(template, `Template '${name}' not found`);
+    await scaffoldNewProject(
+      NodeHost,
+      makeScaffoldingConfig(template, {
+        name,
+        directory: targetFolder,
+        baseUri: typeSpecCoreTemplates.baseUri,
+      }),
+    );
+  }
+  async function scaffoldTemplateSnapshot(name: string): Promise<void> {
+    await scaffoldTemplateTo(name, resolve(snapshotFolder, name));
+  }
+
+  async function scaffoldTemplateForTest(name: string): Promise<ScaffoldedTemplateFixture> {
+    const targetFolder = resolve(testTempRoot, name);
+    await scaffoldTemplateTo(name, targetFolder);
+
+    return {
+      directory: targetFolder,
+      checkCommand: async (command: string, args: string[] = [], options: SpawnOptions = {}) => {
+        const xplatCmd = process.platform === "win32" ? `${command}.cmd` : command;
+        const shell = process.platform === "win32" ? true : options.shell;
+        const result = await execAsync(xplatCmd, args, {
+          shell,
+          ...options,
+          cwd: targetFolder,
+        });
+        ok(
+          result.exitCode === 0,
+          [
+            `Command '${command} ${args.join(" ")}' failed with exit code ${result.exitCode}`,
+            "-".repeat(100),
+            result.stdio,
+            "-".repeat(100),
+          ].join("\n"),
+        );
+      },
+    };
+  }
+
+  describe("create templates", () => {
+    beforeAll(async () => {
+      await rm(snapshotFolder, { recursive: true, force: true });
+    });
+
+    it("rest", () => scaffoldTemplateSnapshot("rest"));
+    it("emitter-ts", () => scaffoldTemplateSnapshot("emitter-ts"));
+    it("library-ts", () => scaffoldTemplateSnapshot("library-ts"));
+  });
+
+  describe("validate templates", () => {
+    it("validate rest template", async () => {
+      const fixture = await scaffoldTemplateForTest("rest");
+      await fixture.checkCommand("npm", ["install"]);
+      await fixture.checkCommand("npx", ["tsp", "compile", "."]);
+    });
+    it("validate emitter-ts template", async () => {
+      const fixture = await scaffoldTemplateForTest("emitter-ts");
+      await fixture.checkCommand("npm", ["install"]);
+      await fixture.checkCommand("npm", ["run", "build"]);
+      await fixture.checkCommand("npm", ["run", "test"]);
+      await fixture.checkCommand("npm", ["run", "lint"]);
+      await fixture.checkCommand("npm", ["run", "format"]);
+    });
+
+    it("validate library-ts template", async () => {
+      const fixture = await scaffoldTemplateForTest("library-ts");
+      await fixture.checkCommand("npm", ["install"]);
+      await fixture.checkCommand("npm", ["run", "build"]);
+      await fixture.checkCommand("npm", ["run", "test"]);
+      await fixture.checkCommand("npm", ["run", "lint"]);
+      await fixture.checkCommand("npm", ["run", "format"]);
+    });
+  });
+});

packages/init-templates/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "dist",
+    "tsBuildInfoFile": ".temp/.tsbuildinfo",
+    "verbatimModuleSyntax": true
+  },
+  "include": [
+    "src/**/*.ts",
+    "test/**/*.ts",
+    "scripts/**/*.ts",
+    "../compiler/src/init/init-template.ts"
+, "src/index.js"  ],
+}