Fix regression with keep-ipynb (#12780)

backport of #12793

- update fileInformationCache based on `keep-ipynb` information
- requires to always pass ProjectContext in ExecuteOptions
- requires to ensure fileInformationCache is not cloned by safeCloneDeep
  This allows us to always get the global fileInformationCache from the project context to get / set information in it from different contexts.
- Add tests for project context and single file context

Author: cderv

news/changelog-1.7.md

@@ -2,6 +2,8 @@
 
 ## In this release
 
+- ([#12780](https://github.com/quarto-dev/quarto-cli/issues/12780)): `keep-ipynb: true` now works again correctly and intermediate `.quarto_ipynb` is not removed.
+
 ## In previous releases
 
 - ([#6607](https://github.com/quarto-dev/quarto-cli/issues/6607)): Add missing beamer template update for beamer theme options: `colorthemeoptions`, `fontthemeoptions`, `innerthemeoptions` and `outerthemeoptions`.

src/command/render/render-contexts.ts

@@ -71,8 +71,6 @@ import {
   kProjectType,
   ProjectContext,
 } from "../../project/types.ts";
-import { isHtmlDashboardOutput, isHtmlOutput } from "../../config/format.ts";
-import { formatHasBootstrap } from "../../format/html/format-html-info.ts";
 import { warnOnce } from "../../core/log.ts";
 import { dirAndStem } from "../../core/path.ts";
 import { fileExecutionEngineAndTarget } from "../../execute/engine.ts";
@@ -88,6 +86,8 @@ import {
 } from "../../core/pandoc/pandoc-formats.ts";
 import { ExtensionContext } from "../../extension/types.ts";
 import { NotebookContext } from "../../render/notebook/notebook-types.ts";
+import { isHtmlDashboardOutput, isHtmlOutput } from "../../config/format.ts";
+import { formatHasBootstrap } from "../../format/html/format-html-info.ts";
 
 export async function resolveFormatsFromMetadata(
   metadata: Metadata,
@@ -297,7 +297,7 @@ export async function renderContexts(
 
     // if this isn't for execute then cleanup context
     if (!forExecute && engine.executeTargetSkipped) {
-      engine.executeTargetSkipped(target, formats[formatKey].format);
+      engine.executeTargetSkipped(target, formats[formatKey].format, project);
     }
   }
   return contexts;

src/command/render/render-files.ts

@@ -199,7 +199,11 @@ export async function renderExecute(
 
         // notify engine that we skipped execute
         if (context.engine.executeTargetSkipped) {
-          context.engine.executeTargetSkipped(context.target, context.format);
+          context.engine.executeTargetSkipped(
+            context.target,
+            context.format,
+            context.project,
+          );
         }
 
         // return results

src/core/jupyter/jupyter-embed.ts

@@ -43,7 +43,13 @@ import {
   JupyterCellOutput,
 } from "../jupyter/types.ts";
 
-import { dirname, extname, join, basename, isAbsolute } from "../../deno_ral/path.ts";
+import {
+  basename,
+  dirname,
+  extname,
+  isAbsolute,
+  join,
+} from "../../deno_ral/path.ts";
 import { languages } from "../handlers/base.ts";
 import {
   extractJupyterWidgetDependencies,
@@ -596,6 +602,7 @@ async function getCachedNotebookInfo(
       quiet: flags.quiet,
       previewServer: context.options.previewServer,
       handledLanguages: languages(),
+      project: context.project,
     };
 
     const [dir, stem] = dirAndStem(nbAddress.path);

src/execute/jupyter/jupyter.ts

@@ -17,7 +17,7 @@ import { readYamlFromMarkdown } from "../../core/yaml.ts";
 import { isInteractiveSession } from "../../core/platform.ts";
 import { partitionMarkdown } from "../../core/pandoc/pandoc-partition.ts";
 
-import { dirAndStem, normalizePath, removeIfExists } from "../../core/path.ts";
+import { dirAndStem, normalizePath } from "../../core/path.ts";
 import { runningInCI } from "../../core/ci-info.ts";
 
 import {
@@ -109,7 +109,10 @@ import {
 import { jupyterCapabilities } from "../../core/jupyter/capabilities.ts";
 import { runExternalPreviewServer } from "../../preview/preview-server.ts";
 import { onCleanup } from "../../core/cleanup.ts";
-import { projectOutputDir } from "../../project/project-shared.ts";
+import {
+  ensureFileInformationCache,
+  projectOutputDir,
+} from "../../project/project-shared.ts";
 import { assert } from "testing/asserts";
 
 export const jupyterEngine: ExecutionEngine = {
@@ -436,7 +439,7 @@ export const jupyterEngine: ExecutionEngine = {
 
     // if it's a transient notebook then remove it
     // (unless keep-ipynb was specified)
-    cleanupNotebook(options.target, options.format);
+    cleanupNotebook(options.target, options.format, options.project);
 
     // Create markdown from the result
     const outputs = result.cellOutputs.map((output) => output.markdown);
@@ -713,12 +716,17 @@ async function disableDaemonForNotebook(target: ExecutionTarget) {
   return false;
 }
 
-function cleanupNotebook(target: ExecutionTarget, format: Format) {
-  // remove transient notebook if appropriate
+function cleanupNotebook(
+  target: ExecutionTarget,
+  format: Format,
+  project: ProjectContext,
+) {
+  // Make notebook non-transient when keep-ipynb is set
   const data = target.data as JupyterTargetData;
-  if (data.transient) {
-    if (!format.execute[kKeepIpynb]) {
-      removeIfExists(target.input);
+  const cached = ensureFileInformationCache(project, target.source);
+  if (data.transient && format.execute[kKeepIpynb]) {
+    if (cached.target && cached.target.data) {
+      (cached.target.data as JupyterTargetData).transient = false;
     }
   }
 }

src/execute/types.ts

@@ -49,7 +49,11 @@ export interface ExecutionEngine {
     format: Format,
   ) => Format;
   execute: (options: ExecuteOptions) => Promise<ExecuteResult>;
-  executeTargetSkipped?: (target: ExecutionTarget, format: Format) => void;
+  executeTargetSkipped?: (
+    target: ExecutionTarget,
+    format: Format,
+    project: ProjectContext,
+  ) => void;
   dependencies: (options: DependenciesOptions) => Promise<DependenciesResult>;
   postprocess: (options: PostProcessOptions) => Promise<void>;
   canFreeze: boolean;
@@ -89,7 +93,7 @@ export interface ExecuteOptions {
   quiet?: boolean;
   previewServer?: boolean;
   handledLanguages: string[]; // list of languages handled by cell language handlers, after the execution engine
-  project?: ProjectContext;
+  project: ProjectContext;
 }
 
 // result of execution

src/project/project-context.ts

@@ -70,6 +70,7 @@ import { projectResourceFiles } from "./project-resources.ts";
 
 import {
   cleanupFileInformationCache,
+  FileInformationCacheMap,
   ignoreFieldsForProjectType,
   normalizeFormatYaml,
   projectConfigFile,
@@ -272,7 +273,7 @@ export async function projectContext(
           dir: join(dir, ".quarto"),
           prefix: "quarto-session-temp",
         });
-        const fileInformationCache = new Map();
+        const fileInformationCache = new FileInformationCacheMap();
         const result: ProjectContext = {
           resolveBrand: async (fileName?: string) =>
             projectResolveBrand(result, fileName),
@@ -368,7 +369,7 @@ export async function projectContext(
           dir: join(dir, ".quarto"),
           prefix: "quarto-session-temp",
         });
-        const fileInformationCache = new Map();
+        const fileInformationCache = new FileInformationCacheMap();
         const result: ProjectContext = {
           resolveBrand: async (fileName?: string) =>
             projectResolveBrand(result, fileName),
@@ -443,7 +444,7 @@ export async function projectContext(
             dir: join(originalDir, ".quarto"),
             prefix: "quarto-session-temp",
           });
-          const fileInformationCache = new Map();
+          const fileInformationCache = new FileInformationCacheMap();
           const context: ProjectContext = {
             resolveBrand: async (fileName?: string) =>
               projectResolveBrand(context, fileName),

src/project/project-shared.ts

@@ -53,6 +53,7 @@ import {
 } from "../resources/types/schema-types.ts";
 import { Brand } from "../core/brand/brand.ts";
 import { assert } from "testing/asserts";
+import { Cloneable } from "../core/safe-clone-deep.ts";
 
 export function projectExcludeDirs(context: ProjectContext): string[] {
   const outputDir = projectOutputDir(context);
@@ -633,6 +634,15 @@ export async function projectResolveBrand(
   }
 }
 
+// Create a class that extends Map and implements Cloneable
+export class FileInformationCacheMap extends Map<string, FileInformation>
+  implements Cloneable<Map<string, FileInformation>> {
+  clone(): Map<string, FileInformation> {
+    // Return the same instance (reference) instead of creating a clone
+    return this;
+  }
+}
+
 export function cleanupFileInformationCache(project: ProjectContext) {
   project.fileInformationCache.forEach((entry) => {
     if (entry?.target?.data) {

src/project/types/single-file/single-file.ts

@@ -21,6 +21,7 @@ import { MappedString } from "../../../core/mapped-text.ts";
 import { fileExecutionEngineAndTarget } from "../../../execute/engine.ts";
 import {
   cleanupFileInformationCache,
+  FileInformationCacheMap,
   projectFileMetadata,
   projectResolveBrand,
   projectResolveFullMarkdownForFile,
@@ -49,7 +50,7 @@ export async function singleFileProjectContext(
     notebookContext,
     environment: () => environmentMemoizer(result),
     renderFormats,
-    fileInformationCache: new Map(),
+    fileInformationCache: new FileInformationCacheMap(),
     fileExecutionEngineAndTarget: (
       file: string,
     ) => {

tests/docs/smoke-all/2025/05/21/keep_ipynb_project/.gitignore

@@ -0,0 +1 @@
+/.quarto/