fix: address various PR concerns and comments

Author: joroscoSF

messages/lineage.md

@@ -4,7 +4,7 @@ Generate field-level dataLineage for a Data Code Extension %s package.
 
 # description
 
-Runs the package's entrypoint against a Salesforce org with every write intercepted, traces each written field back to the fields it was computed from, and replaces the dataLineage section of the package's config.json with the result. Nothing is written to Data Cloud. Run 'scan' first so the read and write permissions in config.json are current. Fields whose lineage can't be traced are listed and get no lineage unless you add entries by hand.
+Runs the package's entrypoint against a Salesforce org with its Data Cloud writes intercepted, traces each written field back to the fields it was computed from, and replaces the dataLineage section of the package's config.json with the result. Nothing is written to Data Cloud objects, but other calls the script makes still run for real with the org's credentials, including named credential requests, LLM gateway calls, and Einstein predictions. Streaming scripts aren't supported. Run 'scan' first so the read and write permissions in config.json are current. Fields whose lineage can't be traced are listed and get no lineage unless you add entries by hand.
 
 Generated lineage is a best-effort analysis of your script and may be incomplete or wrong (for example, for UDFs or other code the tool can't see into). It determines how governance tags propagate, which can restrict or open access to data. You're responsible for reviewing and maintaining it. Before config.json is written, you must accept this, either at the prompt or with --accept-lineage-ownership.
 
@@ -48,12 +48,22 @@ Successfully authenticated with org '%s'
 
 # info.generating
 
-Running package to trace lineage. Nothing is written to Data Cloud...
+Running package to trace lineage. Data Cloud writes are intercepted; other external calls the script makes still run...
 
 # info.summary
 
 Generated %s dataLineage entries with %s (target, source) field pairs: +%s added, -%s removed, ~%s changed.
 
+# info.removed
+
+Entries in config.json that the generated lineage drops:
+%s
+
+# info.changed
+
+Entries in config.json whose source fields the generated lineage changes:
+%s
+
 # info.noSources
 
 %s target fields are computed from no source field (for example, literals) and get no lineage:
@@ -95,13 +105,17 @@ Lineage generation completed successfully!
 
 # warning.permissionGaps
 
-The generated lineage references objects that aren't in config.json permissions, so deploy would fail (1041/1042). Run 'data-code-extension script scan' first, then generate lineage again.
+The generated lineage references objects that aren't in config.json permissions, so deploy would fail (1041/1042). Run 'data-code-extension script scan' first, then generate lineage again. Scan can't detect objects whose names your script builds at run time; add those to permissions in config.json by hand.
 %s
 
 # warning.emptyResult
 
 No lineage could be generated, so config.json was not modified. The package follows the no-lineage path unless you add dataLineage entries by hand.
 
+# warning.emptyResultExisting
+
+No lineage could be generated, so config.json was not modified. Its existing %s dataLineage entries still apply and are deployed with the package. Review or remove them in '%s'.
+
 # warning.replace
 
 This replaces the existing dataLineage section. Any entries you added or edited by hand will be lost. Lineage determines how governance tags propagate to the target; review the result before deploying.
@@ -126,6 +140,14 @@ Accept and replace the existing dataLineage in config.json?
 
 Lineage can only be generated for script packages; '%s' is a function package.
 
+# error.streamingPackage
+
+Lineage can't be generated for streaming scripts; '%s' has a streamingSource in config.json.
+
+# error.configUnwritable
+
+Can't write '%s': %s
+
 # error.outputFileCollision
 
 --output-file '%s' is inside the package's payload or project metadata, or is one of its config files. Choose a path outside the package's payload directory.
@@ -152,8 +174,8 @@ Writing generated lineage to config.json requires accepting ownership of it.
 
 # actions.ownershipNotAccepted
 
-- Review the generated lineage, then re-run with --accept-lineage-ownership to accept responsibility for it and write it to config.json.
-- Re-run with --dry-run to preview the generated lineage without changing config.json.
+- Re-run with --accept-lineage-ownership to accept responsibility for the generated lineage and write it to config.json.
+- Re-run with --dry-run to preview the generated lineage, or check it against config.json, without writing.
 
 # error.replaceNotConfirmed
 
@@ -162,7 +184,7 @@ config.json already has a different dataLineage section, and replacing it needs
 # actions.replaceNotConfirmed
 
 - Re-run with --no-prompt to replace the existing dataLineage.
-- Re-run with --dry-run to preview the generated lineage without changing config.json.
+- Re-run with --dry-run to preview the generated lineage, or check it against config.json, without writing.
 
 # error.sdkTooOld
 
@@ -203,7 +225,7 @@ Target Salesforce org to read data from.
 
 # flags.targetOrg.description
 
-The alias or username of the Salesforce org whose data the package reads while lineage is traced. The org must have Data Cloud enabled and appropriate permissions. Nothing is written to the org.
+The alias or username of the Salesforce org whose data the package reads while lineage is traced. The org must have Data Cloud enabled and appropriate permissions. Nothing is written to Data Cloud objects in the org, but other calls the script makes, such as named credential requests, LLM gateway calls, and Einstein predictions, still run against it.
 
 # flags.configFile.summary
 
@@ -215,11 +237,11 @@ Optional path to an SDK config.yaml that overrides the reader and writer configu
 
 # flags.dependencies.summary
 
-Dependencies override for the run.
+Modules to import before the run.
 
 # flags.dependencies.description
 
-Optional comma-separated list of Python package dependencies to use during the run, overriding those defined in the package's requirements.txt.
+Optional comma-separated list of Python modules (or module.attribute) to import before the entrypoint runs. Use import names, such as 'yaml' rather than 'pyyaml'. Nothing is installed: each module must already be installed in the active Python environment.
 
 # flags.dryRun.summary
 
@@ -243,12 +265,12 @@ Replace existing dataLineage without prompting.
 
 # flags.noPrompt.description
 
-Skips the confirmation before replacing a dataLineage section that differs from the generated one. Required to replace existing lineage when running non-interactively or with --json. Doesn't accept ownership of the generated lineage; use --accept-lineage-ownership for that.
+Skips the confirmation before replacing a dataLineage section that differs from the generated one. Required when running non-interactively or with --json and config.json already has dataLineage; checked before the run. Doesn't accept ownership of the generated lineage; use --accept-lineage-ownership for that.
 
 # flags.acceptLineageOwnership.summary
 
 Accept responsibility for the generated lineage without prompting.
 
 # flags.acceptLineageOwnership.description
 
-Confirms that you understand the generated lineage is best-effort and may be incomplete or wrong, that it determines how governance tags propagate, and that you're responsible for reviewing and maintaining it. Skips the ownership prompt. Required to write lineage when running non-interactively or with --json.
+Confirms that you understand the generated lineage is best-effort and may be incomplete or wrong, that it determines how governance tags propagate, and that you're responsible for reviewing and maintaining it. Skips the ownership prompt. Required when running non-interactively or with --json, unless you use --dry-run; checked before the run.

messages/run.md

@@ -98,6 +98,15 @@ Authentication with Salesforce org '%s' failed while running the package.
 - Confirm the org alias or username passed to --target-org is correct.
 - Verify the org has Data Cloud enabled and your user has the required permissions.
 
+# error.runTerminated
+
+The package run was stopped by signal %s before it finished. Runs time out after %s seconds.
+
+# actions.runTerminated
+
+- If the run timed out, reduce the data it processes (for example, the reader's default_row_limit in an SDK config file) or check for a step that hangs, such as a network call.
+- If the process ran out of memory or was stopped by another program, free resources and try again.
+
 # error.runProcessStartFailed
 
 Could not start the Python interpreter '%s': %s
@@ -149,8 +158,8 @@ Optional path to a JSON config file that provides input payload for the run. Def
 
 # flags.dependencies.summary
 
-Dependencies override for the run.
+Modules to import before the run.
 
 # flags.dependencies.description
 
-Optional comma-separated list of Python package dependencies to use during the run, overriding those defined in the package's requirements.txt.
+Optional comma-separated list of Python modules (or module.attribute) to import before the entrypoint runs. Use import names, such as 'yaml' rather than 'pyyaml'. Nothing is installed: each module must already be installed in the active Python environment.

messages/scan.md

@@ -119,9 +119,9 @@ dataLineage references objects that are no longer in permissions (%s entries aff
 
 Write the updated permissions to config.json anyway?
 
-# info.staleLineageNotWritten
+# error.staleLineageNotWritten
 
-No files were modified. Update dataLineage, then scan again.
+The updated permissions weren't confirmed, so no files were modified. Update dataLineage, then scan again.
 
 # error.staleLineageNotConfirmed
 

src/base/lineageBase.ts

@@ -87,7 +87,20 @@ export abstract class LineageBase extends SfCommand<LineageGenerateResult> {
         throw new SfError(messages.getMessage('error.functionPackage', [flags.entrypoint]), 'LineageFunctionPackage');
       }
       // Read config.json up front so a missing or malformed file fails before the run.
-      await readConfigJson(configPath);
+      const initialConfig = await readConfigJson(configPath);
+      if (initialConfig.streamingSource) {
+        throw new SfError(messages.getMessage('error.streamingPackage', [flags.entrypoint]), 'LineageStreamingPackage');
+      }
+      // Fail before the run on everything that would only refuse the write afterwards.
+      if (!flags['dry-run']) {
+        this.checkNonInteractiveWrite(messages, flags, hasLineageSection(initialConfig));
+        await fs.access(configPath, fsConstants.W_OK).catch((err: Error) => {
+          throw new SfError(
+            messages.getMessage('error.configUnwritable', [configPath, err.message]),
+            'LineageConfigUnwritable'
+          );
+        });
+      }
 
       const outputFile = flags['output-file'] ? path.resolve(flags['output-file']) : undefined;
       if (outputFile) {
@@ -112,7 +125,7 @@ export abstract class LineageBase extends SfCommand<LineageGenerateResult> {
           );
         }
         // Fail before the run rather than lose its result to a bad path afterwards.
-        await fs.access(path.dirname(outputFile), fsConstants.W_OK).catch((err: Error) => {
+        await checkOutputFileWritable(outputFile).catch((err: Error) => {
           throw new SfError(
             messages.getMessage('error.outputFileUnwritable', [outputFile, err.message]),
             'LineageOutputFileUnwritable'
@@ -179,7 +192,12 @@ export abstract class LineageBase extends SfCommand<LineageGenerateResult> {
       };
 
       if (entries.length === 0) {
-        this.warn(messages.getMessage('warning.emptyResult'));
+        // Existing entries still ship at deploy; say so rather than imply the no-lineage path.
+        this.warn(
+          current.length > 0
+            ? messages.getMessage('warning.emptyResultExisting', [current.length, configPath])
+            : messages.getMessage('warning.emptyResult')
+        );
         return result;
       }
 
@@ -233,6 +251,12 @@ export abstract class LineageBase extends SfCommand<LineageGenerateResult> {
         diff.changed.length,
       ])
     );
+    if (diff.removed.length > 0) {
+      this.log(messages.getMessage('info.removed', [bullets(diff.removed)]));
+    }
+    if (diff.changed.length > 0) {
+      this.log(messages.getMessage('info.changed', [bullets(diff.changed)]));
+    }
     if (lineage.noSources.length > 0) {
       this.log(messages.getMessage('info.noSources', [lineage.noSources.length, bullets(lineage.noSources)]));
     }
@@ -277,13 +301,36 @@ export abstract class LineageBase extends SfCommand<LineageGenerateResult> {
       );
     }
     const prompt = needsOwnership
-      ? replacing
+      ? needsReplace
         ? 'prompt.acceptOwnershipReplace'
         : 'prompt.acceptOwnership'
       : 'prompt.replace';
     return this.confirm({ message: messages.getMessage(prompt), defaultAnswer: false, ms: LINEAGE_CONFIRM_TIMEOUT_MS });
   }
 
+  /**
+   * Without a prompt to fall back on, a write needs its flags up front: fail before the run
+   * instead of after it. `--dry-run` is the way to check lineage without writing.
+   * confirmWrite repeats these checks in case config.json gains a dataLineage section during the run.
+   */
+  private checkNonInteractiveWrite(messages: Messages<string>, flags: BaseLineageFlags, replacing: boolean): void {
+    if (!this.jsonEnabled() && process.stdin.isTTY) return;
+    if (!flags['accept-lineage-ownership']) {
+      throw new SfError(
+        messages.getMessage('error.ownershipNotAccepted'),
+        'LineageOwnershipNotAccepted',
+        messages.getMessages('actions.ownershipNotAccepted')
+      );
+    }
+    if (replacing && !flags['no-prompt']) {
+      throw new SfError(
+        messages.getMessage('error.replaceNotConfirmed'),
+        'LineageReplaceNotConfirmed',
+        messages.getMessages('actions.replaceNotConfirmed')
+      );
+    }
+  }
+
   protected abstract getCodeType(): 'script';
   protected abstract getMessages(): Messages<string>;
 }
@@ -356,6 +403,20 @@ async function realPath(file: string, hops = 0): Promise<string> {
   }
 }
 
+/**
+ * Rejects an --output-file that `fs.writeFile` would fail on: an existing target must be a
+ * writable regular file, and a new one needs a writable parent directory.
+ */
+async function checkOutputFileWritable(outputFile: string): Promise<void> {
+  const stat = await statOrUndefined(outputFile);
+  if (stat === undefined) {
+    await fs.access(path.dirname(outputFile), fsConstants.W_OK);
+    return;
+  }
+  if (!stat.isFile()) throw new Error('not a regular file');
+  await fs.access(outputFile, fsConstants.W_OK);
+}
+
 function bullets(lines: string[]): string {
   return lines.map((line) => `  - ${line}`).join('\n');
 }

src/base/scanBase.ts

@@ -77,17 +77,13 @@ export abstract class ScanBase extends SfCommand<ScanResult> {
         if (hasStaleLineage(stale)) this.warnStaleLineage(messages, executionResult.config, stale);
       }
 
+      // Exit non-zero so a following deploy doesn't ship the old permissions.
       if (executionResult.written === false) {
-        this.log(messages.getMessage('info.staleLineageNotWritten'));
-        return {
-          success: false,
-          pythonInfo,
-          packageInfo,
-          codeType,
-          workingDirectory: workingDir,
-          executionResult,
-          message: messages.getMessage('info.staleLineageNotWritten'),
-        };
+        throw new SfError(
+          messages.getMessage('error.staleLineageNotWritten'),
+          'ScanStaleLineageNotWritten',
+          messages.getMessages('actions.staleLineageNotConfirmed')
+        );
       }
 
       this.log(messages.getMessage('info.scanExecuted', [workingDir]));

src/utils/lineageConfig.ts

@@ -152,9 +152,9 @@ export function countEntriesReferencing(config: Record<string, unknown>, objects
   ).length;
 }
 
-/** Write config.json with a 2-space indent and a trailing newline. */
+/** Write config.json with a 2-space indent and no trailing newline, as scan and init do. */
 export async function writeConfigJson(configPath: string, config: Record<string, unknown>): Promise<void> {
-  await fs.writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, 'utf8');
+  await fs.writeFile(configPath, JSON.stringify(config, null, 2), 'utf8');
 }
 
 function objectOf(field: string): string {