Add opt-in incremental (append-semantics) element data delivery

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sigmacomputing/plugin-sdk-root",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "private": true,
   "description": "Sigma Computing Plugin Client SDK",
   "license": "MIT",

packages/plugin-sdk/README.md

@@ -795,6 +795,70 @@ interface WorkbookElementData {
 }
 ```
 
+#### useIncrementalElementData()
+
+Drop-in replacement for `usePaginatedElementData()` that opts in to
+incremental (append-semantics) data delivery. When the host supports it, each
+page is delivered as a chunk containing only the new rows, so loading a large
+element costs each row once instead of re-delivering the entire accumulated
+data set on every page. When the host does not support incremental delivery,
+the hook transparently falls back to today's cumulative behavior — no
+branching code is required in the plugin.
+
+```ts
+function useIncrementalElementData(
+  configId: string,
+): [WorkbookElementData, () => void, IncrementalElementDataInfo];
+```
+
+Arguments
+
+- `configId : string` - A workbook element’s unique identifier from the plugin config.
+
+Returns the accumulated row data from the specified element, a callback for
+fetching more data, and progress metadata:
+
+```ts
+interface IncrementalElementDataInfo {
+  rowCount: number; // rows accumulated so far
+  isComplete: boolean; // true once the host reports no more rows
+  totalRows?: number; // total rows in the source element, if the host reports it
+}
+```
+
+> **Warning:** on hosts without incremental support, `isComplete` stays
+> `false` forever — completion is a signal only incremental-capable hosts can
+> send. Never drive an auto-load loop or a "load more" affordance from
+> `isComplete` alone; use `rowCount` to detect whether a fetch actually made
+> progress (if it stops growing, there is no more data).
+
+Example
+
+```ts
+const [data, loadMore, { rowCount, isComplete }] =
+  useIncrementalElementData('source');
+```
+
+Framework Agnostic Usage
+
+```ts
+const unsubscribe = client.elements.subscribeToIncrementalElementData(
+  'source',
+  chunk => {
+    // chunk.data contains only this chunk's rows; chunk.offset is the
+    // absolute row offset to apply them at. Hosts without incremental
+    // support deliver their cumulative payloads as replace-everything
+    // chunks at offset 0.
+    applyRowsAtOffset(chunk.data, chunk.offset);
+  },
+);
+```
+
+Use one subscription style per element: the delivery mode belongs to the
+(plugin, element) subscription, so mixing `subscribeToElementData` and
+`subscribeToIncrementalElementData` (or their hooks) on the same config
+element is unsupported.
+
 #### useVariable()
 
 Returns a given variable's value and a setter to update that variable

packages/plugin-sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sigmacomputing/plugin",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Sigma Computing Plugin Client SDK",
   "license": "MIT",
   "type": "module",

packages/plugin-sdk/src/client/__tests__/initialize.test.ts

@@ -638,6 +638,105 @@ describe('initialize', () => {
       expect(callback).not.toHaveBeenCalled();
     });
 
+    it('subscribeToIncrementalElementData subscribes with the incremental capability, dispatches chunks, and unsubscribes', () => {
+      const callback = vi.fn();
+      const unsub = client.elements.subscribeToIncrementalElementData(
+        'el1',
+        callback,
+      );
+
+      const sub = findPostMessage(
+        postMessageSpy,
+        'wb:plugin:element:subscribe:data',
+      );
+      expect(sub?.data.args).toEqual(['el1', { mode: 'incremental' }]);
+
+      const chunk = {
+        data: { c1: [1, 2, 3] },
+        offset: 0,
+        isComplete: false,
+        totalRows: 6,
+      };
+      sendWindowMessage({
+        type: 'wb:plugin:element:el1:data',
+        result: chunk,
+        error: null,
+      });
+      expect(callback).toHaveBeenCalledWith(chunk);
+
+      postMessageSpy.mockClear();
+      callback.mockClear();
+      unsub();
+      const unsubMsg = findPostMessage(
+        postMessageSpy,
+        'wb:plugin:element:unsubscribe:data',
+      );
+      expect(unsubMsg?.data.args).toEqual(['el1']);
+
+      sendWindowMessage({
+        type: 'wb:plugin:element:el1:data',
+        result: chunk,
+        error: null,
+      });
+      expect(callback).not.toHaveBeenCalled();
+    });
+
+    it('subscribeToIncrementalElementData normalizes legacy cumulative payloads into replace chunks at offset 0', () => {
+      const callback = vi.fn();
+      client.elements.subscribeToIncrementalElementData('el1', callback);
+
+      const legacyData = { c1: [1, 2, 3], c2: ['a', 'b', 'c'] };
+      sendWindowMessage({
+        type: 'wb:plugin:element:el1:data',
+        result: legacyData,
+        error: null,
+      });
+      expect(callback).toHaveBeenCalledWith({
+        data: legacyData,
+        offset: 0,
+        isComplete: false,
+      });
+    });
+
+    it('subscribeToIncrementalElementData normalizes a null payload into an empty replace chunk', () => {
+      const callback = vi.fn();
+      client.elements.subscribeToIncrementalElementData('el1', callback);
+
+      // The host sends null when the element's data eval fails.
+      sendWindowMessage({
+        type: 'wb:plugin:element:el1:data',
+        result: null,
+        error: null,
+      });
+      expect(callback).toHaveBeenCalledWith({
+        data: {},
+        offset: 0,
+        isComplete: false,
+      });
+    });
+
+    it('subscribeToIncrementalElementData treats envelopes with malformed offsets as legacy payloads', () => {
+      const callback = vi.fn();
+      client.elements.subscribeToIncrementalElementData('el1', callback);
+
+      for (const offset of [-1, 1.5, Number.NaN]) {
+        callback.mockClear();
+        const malformed = { data: { c1: [1] }, offset, isComplete: true };
+        sendWindowMessage({
+          type: 'wb:plugin:element:el1:data',
+          result: malformed,
+          error: null,
+        });
+        // Not recognized as a chunk: falls back to replace-at-0 normalization
+        // instead of corrupting chunk assembly downstream.
+        expect(callback).toHaveBeenCalledWith({
+          data: malformed,
+          offset: 0,
+          isComplete: false,
+        });
+      }
+    });
+
     it('fetchMoreElementData posts wb:plugin:element:fetch-more', () => {
       client.elements.fetchMoreElementData('el1');
       const msg = findPostMessage(

packages/plugin-sdk/src/client/initialize.ts

@@ -4,11 +4,33 @@ import {
   PluginMessageResponse,
   PluginStyle,
   UrlParameter,
+  WorkbookElementData,
+  WorkbookElementDataChunk,
   WorkbookSelection,
   WorkbookVariable,
 } from '../types';
 import { validateConfigId } from '../utils/error';
 
+// Every value in a legacy cumulative WorkbookElementData payload is a column
+// array, so typed non-array `offset`/`isComplete`/`data` fields can only come
+// from the incremental chunk envelope. Offsets must be non-negative integers;
+// a payload with a malformed offset is treated as legacy data rather than
+// letting a NaN/negative/fractional value corrupt chunk assembly downstream.
+function isElementDataChunk(
+  result: unknown,
+): result is WorkbookElementDataChunk {
+  if (typeof result !== 'object' || result === null) return false;
+  const chunk = result as Partial<WorkbookElementDataChunk>;
+  return (
+    Number.isInteger(chunk.offset) &&
+    (chunk.offset as number) >= 0 &&
+    typeof chunk.isComplete === 'boolean' &&
+    typeof chunk.data === 'object' &&
+    chunk.data !== null &&
+    !Array.isArray(chunk.data)
+  );
+}
+
 export function initialize<T = {}>(): PluginInstance<T> {
   const pluginConfig: Partial<PluginConfig<T>> = {
     config: {} as T,
@@ -255,6 +277,38 @@ export function initialize<T = {}>(): PluginInstance<T> {
           void execPromise('wb:plugin:element:unsubscribe:data', configId);
         };
       },
+      subscribeToIncrementalElementData(configId, callback) {
+        validateConfigId(configId, 'element');
+        const eventName = `wb:plugin:element:${configId}:data`;
+        const onData = (result: unknown) => {
+          if (isElementDataChunk(result)) {
+            callback(result);
+          } else {
+            // A host without incremental support ignores the subscribe
+            // options and keeps sending cumulative payloads. Deliver those as
+            // replace-everything chunks so consumers behave identically
+            // against either host. A host also sends null when the element's
+            // data eval fails, which normalizes to an empty chunk so it
+            // degrades the same way the non-incremental subscription does
+            // rather than throwing during chunk assembly. Legacy hosts never
+            // signal completion, so isComplete stays false.
+            callback({
+              data: (result ?? {}) as WorkbookElementData,
+              offset: 0,
+              isComplete: false,
+            });
+          }
+        };
+        on(eventName, onData);
+        void execPromise('wb:plugin:element:subscribe:data', configId, {
+          mode: 'incremental',
+        });
+
+        return () => {
+          off(eventName, onData);
+          void execPromise('wb:plugin:element:unsubscribe:data', configId);
+        };
+      },
       fetchMoreElementData(configId) {
         validateConfigId(configId, 'element');
         void execPromise('wb:plugin:element:fetch-more', configId);