From 4d35052fe9fd860fce0c05601e806a016a692669 Mon Sep 17 00:00:00 2001 From: Hannah Bulmer <35310862+hannah-bulmer@users.noreply.github.com> Date: Mon, 28 Oct 2019 17:17:35 +0100 Subject: [PATCH 01/10] Add specification to lookup (at most 1) (#5) --- .../StandardizedActionsAndTriggers.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md index abba514..134dc34 100644 --- a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md +++ b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md @@ -1,8 +1,8 @@ # Descriptions of standardized actions or triggers -**Version Publish Date:** 07.11.2019 +**Version Publish Date:** 28.10.2019 -**Semantic Version of Document:** 2.2.1 +**Semantic Version of Document:** 2.2.2 ## Table of Contents @@ -115,8 +115,8 @@ I have a contact who works for a company. I have an ID or other distinguishing ##### Config Fields - Object Type (dropdown) -- Allow ID to be omitted (dropdown/checkbox: yes/no) -- Allow zero results (dropdown/checkbox: yes/no) +- Allow ID to be omitted (dropdown/checkbox: yes/no); when selected, the ID field becomes optional, otherwise it is a required field +- Allow zero results (dropdown/checkbox: yes/no); hen selected, if zero results are returned, the empty object `{}` is emitted, otherwise typically an error would be thrown ##### Input Metadata From 3a0ac16895a5179d397bd2a7c0cf2fdc940814bb Mon Sep 17 00:00:00 2001 From: Pavel Voropaiev Date: Mon, 9 Dec 2019 14:45:38 +0200 Subject: [PATCH 02/10] Number of search terms could==0 for Lookup objects --- .../StandardizedActionsAndTriggers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md index 134dc34..4469115 100644 --- a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md +++ b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md @@ -203,7 +203,7 @@ I want to search my CRM for data based on some criteria. - Object Type (dropdown) - Behavior (dropdown: Fetch all, Fetch Page, Emit Individually) -- Number of search terms (text field: integer >= 1) (iteration 2) +- Number of search terms (text field: integer >= 0) (iteration 2) ##### Input Metadata From 04499578785adfbbd32d8b9904c2fb74ace54efc Mon Sep 17 00:00:00 2001 From: Jacob Horbulyk Date: Tue, 18 Feb 2020 11:40:07 +0100 Subject: [PATCH 03/10] Feb 2020 (#7) --- .../StandardizedActionsAndTriggers.md | 231 ++++++++++++------ 1 file changed, 162 insertions(+), 69 deletions(-) diff --git a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md index 4469115..881d789 100644 --- a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md +++ b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md @@ -1,8 +1,8 @@ # Descriptions of standardized actions or triggers -**Version Publish Date:** 28.10.2019 +**Version Publish Date:** 18.02.2020 -**Semantic Version of Document:** 2.2.2 +**Semantic Version of Document:** 2.3.1 ## Table of Contents @@ -24,6 +24,8 @@ - [Triggers](#triggers) * [Get New and Updated Objects Polling](#get-new-and-updated-objects-polling) * [Webhooks](#webhooks) + * [Get Recently Deleted Objects Polling](#get-recently-deleted-objects-polling) + * [Event Subscription](#event-subscription) * [Bulk Extract](#bulk-extract) It is important to define common rules on how an adapter responds to changes @@ -108,7 +110,7 @@ I have some contact data that I want to add to my CRM. I don't necessarily know ### Lookup Object (at most 1) ##### Example Use Case -I have a contact who works for a company. I have an ID or other distinguishing characteristic (e.g. legal name) of the company and I want to learn some detail about the company (e.g. number of employees). +I have a contact who works for a company. I have an ID or other distinguishing characteristic (e.g. legal name) of the company and I want to learn some detail about the company (e.g. country of company). #### Iteration 1: Lookup Object By ID @@ -116,7 +118,9 @@ I have a contact who works for a company. I have an ID or other distinguishing - Object Type (dropdown) - Allow ID to be omitted (dropdown/checkbox: yes/no); when selected, the ID field becomes optional, otherwise it is a required field -- Allow zero results (dropdown/checkbox: yes/no); hen selected, if zero results are returned, the empty object `{}` is emitted, otherwise typically an error would be thrown +- Allow zero results (dropdown/checkbox: yes/no); When selected, if zero results are returned, the empty object `{}` is emitted, otherwise typically an error would be thrown. +- Wait for object to exist (dropdown/checkbox: yes/no); When selected, if no results are found, apply rebounds and wait until the object exits. +- Linked objects to populate (optional, multi-select dropdown). Select which linked objects to fetch if supported by the API. ##### Input Metadata @@ -135,10 +139,12 @@ I have a contact who works for a company. I have an ID or other distinguishing } try { - const foundObject = GetObjectById(id); // Usually GET verb + const foundObject = GetObjectById(id, linkedObjectsToPopulate); // Usually GET verb emitData(foundObject); } catch (NotFoundException e) { - if(allowZeroResults) { + if(waitForObjectToExist && notAllReboundsExhausted) { + emitRebound({}); + } else if(allowZeroResults) { emitData({}); } else { throw e; @@ -154,10 +160,6 @@ I have a contact who works for a company. I have an ID or other distinguishing - Make sure to Url Encode IDs appearing in HTTP urls -##### Not defined now - -- How to handle populating linked objects. - #### Iteration 2: Lookup Object By Unique Criteria ##### Additional Config Fields @@ -180,9 +182,11 @@ I have a contact who works for a company. I have an ID or other distinguishing } } - const foundObjects = GetObjectsByCriteria(uniqueCriteria); // Usually GET verb + const foundObjects = GetObjectsByCriteria(uniqueCriteria, linkedObjectsToPopulate); // Usually GET verb if(foundObjects.length == 0) { - if(allowZeroResults) { + if(waitForObjectToExist && notAllReboundsExhausted) { + emitRebound({}); + } else if(allowZeroResults) { emitData({}); } else { throw new Error('Not found'); @@ -203,7 +207,8 @@ I want to search my CRM for data based on some criteria. - Object Type (dropdown) - Behavior (dropdown: Fetch all, Fetch Page, Emit Individually) -- Number of search terms (text field: integer >= 0) (iteration 2) +- Number of search terms (text field: integer >= 0) (iteration 2) (0 indicates return all items) +- Linked objects to populate (optional, multi-select dropdown). Select which linked objects to fetch if supported by the API. ##### Input Metadata @@ -223,20 +228,20 @@ I want to search my CRM for data based on some criteria. function lookupObjects(criteria) { switch(mode) { case 'fetchAll': - const results = GetObjectsByCriteria(criteria); + const results = GetObjectsByCriteria(criteria, linkedObjectsToPopulate); if(results.length >= maxResultSize) { throw new Error('Too many results'); } emitData({results: results}); break; case 'emitIndividually': - const results = GetObjectsByCriteria(criteria); + const results = GetObjectsByCriteria(criteria, linkedObjectsToPopulate); results.forEach(result => { emitData(result); } break; case 'fetchPage': - const results = GetObjectsByCritieria(criteria, top: pageSize, skip: pageSize * pageNumber, orderBy: orderByTerms); + const results = GetObjectsByCritieria(criteria, top: pageSize, skip: pageSize * pageNumber, orderBy: orderByTerms, linkedObjectsToPopulate); emitData({results: results}); break; } @@ -254,6 +259,7 @@ I want to search my CRM for data based on some criteria. - Order of operations in multiple terms - How to get total number of matching objects +- How to handle variable number of search terms (perhaps integrator mode?) ### Delete Object ##### Example Use Case @@ -320,14 +326,18 @@ I know the ID of a customer that I want to delete. A simple action to allow integrators to assemble requests to be sent to the system. The component should expose the parts that vary in a typical request. The component should handle authentication and error reporting. +Additional Options to Consider: +* Consider that it may make sense to turn off error reporting & to return things like the HTTP status code & headers +* Consider that it may make sense to allow the option to make a series of sequential array requests. + ##### Example Use Case I'm a technically advanced user who wants to interact with a system in a way not permissible by the existing component actions but would like some simplification relative to using the REST component. ### Lookup Set Of Objects By Unique Criteria -Given an array of information where each item in the array uniquely describes exactly one object. It can be assumed that the array is short enough to reasonably fit the results in a single message. +Given an array of information where each item in the array uniquely describes exactly one object. It can be assumed that the array is short enough to reasonably fit the results in a single message. If any of the objects are not found then it indicates a logic problem in the integration. ##### Example Use Case -I salesperson is responsible for 0 to N accounts. I would like to look up a piece of information for each account associated with the salesperson. +I salesperson is responsible for 0 to N accounts (N being reasonably small). I would like to look up a piece of information for each account associated with the salesperson. #### Iteration 1: Lookup Object By ID #### Iteration 2: Lookup Object By Unique Criteria @@ -335,6 +345,8 @@ I salesperson is responsible for 0 to N accounts. I would like to look up a pie ##### Config Fields - Object Type (dropdown) +- Linked objects to populate (optional, multi-select dropdown). Select which linked objects to fetch if supported by the API. +- Wait for object to exist (dropdown/checkbox: yes/no); When selected, if no results are found, apply rebounds and wait until the object exits. - Iteration 2: Unique Criteria (dropdown) ##### Input Metadata @@ -345,9 +357,13 @@ I salesperson is responsible for 0 to N accounts. I would like to look up a pie function lookupSetOfObjects(itemUniqueCriteriaListToLookup) { const results = itemUniqueCriteriaListToLookup.map(itemUniqueCriteria => { - const matchingItems = GetObjectsByCriteria(itemUniqueCriteria); + const matchingItems = GetObjectsByCriteria(itemUniqueCriteria, linkedObjectsToPopulate); if(matchingItems.length != 1) { - throw new Error(`Lookup failed for ${itemUniqueCriteria}`); + if(!waitForObjectToExist) { + throw new NotFoundError(); + } + emitRebound({}); + return; } return { key: itemCriteria, @@ -365,13 +381,17 @@ I salesperson is responsible for 0 to N accounts. I would like to look up a pie return; } - const searchResults = FetchObjectsWhereIdIn(itemIdsToLookup); + const searchResults = FetchObjectsWhereIdIn(itemIdsToLookup, linkedObjectsToPopulate); const resultDictionary = {}; for each (let itemId of itemIdsToLookup) { const matchingItems = searchResults.filter(result.Id = itemId); if(matchingItems.length != 1) { - throw new Error(`Lookup failed for ${itemUniqueCriteria}`); + if(!waitForObjectToExist) { + throw new NotFoundError(); + } + emitRebound({}); + return; } resultDictionary[itemId] = matchingItems[0]; } @@ -387,10 +407,6 @@ I salesperson is responsible for 0 to N accounts. I would like to look up a pie ##### Gotcha’s to lookout for - Make sure to Url Encode IDs appearing in HTTP urls - -##### Not defined now -- Encode any IDs in URLs -- Rebounds when an object is not found - There are different structures depending on the input structure ### Update Object @@ -399,6 +415,7 @@ I salesperson is responsible for 0 to N accounts. I would like to look up a pie - We will not create the object if it does not exist - The ID/other unique criteria is required - No other fields are required +If the object is not found, then rebounds should be done based on the rebound option. ##### Example Use Case I want to update the price of a product based on its SKU but I don't want to look up other required attributes such as name since I know those have already been set and are not changing. @@ -421,18 +438,31 @@ See above. - the types of the two objects - two sets of unique criteria which describe the two objects - Information about the relationship (e.g. if assigning user to company membership, identify the role of the user) +- There should be an option to emit rebounds should be emitted if results aren't found. ``` function linkObjects(obj1, obj2, linkMetadata) { const matchingObjects1 = lookupObjectByCriteria(obj1.type, obj1.uniqueCriteria); - if (matchingObjects1.length != 1) { - throw new Error('Not found/too many found.'); - } + if (matchingObjects1.length > 1) { + throw new Error('Too many found.'); + } else if (matchingObjects1.length == 0) { + if(!waitForObjectToExist) { + throw new NotFoundError(); + } + emitRebound(); + return; + } const object1Id = matchingObjects1[0].id; const matchingObjects2 = lookupObjectByCriteria(obj2.type, obj2.uniqueCriteria); - if (matchingObjects2.length != 1) { - throw new Error('Not found/too many found.'); + if (matchingObjects2.length > 1) { + throw new Error('Too many found.'); + } else if (matchingObjects2.length == 0) { + if(!waitForObjectToExist) { + throw new NotFoundError(); + } + emitRebound(); + return; } const object2Id = matchingObjects2[0].id; @@ -444,7 +474,7 @@ See above. A student can be a participant in a class and a class can have many students. Given a student ID and a course ID I want to enroll that student in that course. ### Execute Query or Statement in Query Language -Examples of this include constructing a query or statement in SQL, Salesforce’s SOQL, etc. Queries return a table of data when executed. Statements do not reutrn results (other than execution statistics). +Examples of this include constructing a query or statement in SQL, Salesforce’s SOQL, etc. Queries return a table of data when executed. Statements do not return results (other than execution statistics). ##### Example Use Case Execute SQL query in SQL database @@ -586,78 +616,140 @@ I want to learn about changes to contacts in my CRM when they happen. ##### Config Fields - Object Type (dropdown) -- Start Time (string, optional): Indicates the beginning time to start polling from (defaults to the begining of time) +- Start Time (string, optional): Indicates the beginning time to start polling from (defaults to the beginning of time) - End Time (string, optional): If provided, don’t fetch records modified after this time (defaults to never) - Size of Polling Page (optional; positive integer) Indicates the size of pages to be fetched. Defaults to 1000. - Single Page per Interval (dropdown/checkbox: yes/no; default yes) Indicates that if the number of changed records exceeds the maximum number of results in a page, instead of fetching the next page immediately, wait until the next flow start to fetch the next page. +- Time stamp field to poll on (dropdown: created or modified). Indicates just new items or new and modified items. ##### Input Metadata N/A +##### Gotcha’s to lookout for + +- If `previousLastModified` is set to `lastSeenTime` and we have `lastModified >= previousLastModified` then each execution will include records from previous execution. But if at the first execution `previousLastModified` could be equal `cfg.startTime` and we have `lastModified > previousLastModified` then we will lose objects whose last modified date is equal to the `cfg.startTime`. This is why we compare `previousLastModified` and `cfg.startTime || new Date(0)` and if they are equal, use condition `lastModified >= previousLastModified,` else: `lastModified > previousLastModified,` +- We use `lastModified <= maxTime` as it is more understandable for user. +- We have `Single Page per Interval` default to yes because it is slightly safer. +- We need to be careful about more than a page worth of records having the same timestamp. +- We need to be careful about the last record on one page having the same timestamp as the first record on the next page +- We need to be careful about records on page N being modified before reading page N+1 (thus causing records to be skipped as they move from page N+1 to page N). + +##### Assumptions About Server Behavior: +In order for the bellow polling algorithm to work, all of the following must be true about the way the server behaves: +* It is possible to order results by last modified and then by primary key. +* If record A has a timestamp of X and appears within a search but not record B, then if record B appears in a later search than record B MUST have a timestamp that is later (i.e. Not the same or earlier) than record A. + ##### Pseudo-Code +**High level steps:** +1. Retrieve a page of data. +2. If the size of the page is less than the max page size, emit the timestamp of the last record on the page. +3. Compare the timestamps of the last and second last items on the page. If they are different, store the timestamp of the second last record of the page and don't emit the last record. +4. If the timestamps are the same, then store that timestamp and the primary key of the last item. On the next iteration fetch page where the timestamps are equal and primary key is larger than last seen item. + +``` function getObjectsPolling(cfg, snapshot) { - const previousLastModified = snapshot.previousLastModified || cfg.startTime || new Date(0); - const maxTime = cfg.endTime || Date.MaxDate(); - let hasMorePages = true; - snapshot.pageNumber = snapshot.pageNumber || 0; - let lastSeenTime = previousLastModified; + const pollingField = cfg.timeStampFieldToPollOn; + let attemptMorePages = !cfg.singlePagePerInterval; do { + const previousLastModified = snapshot.previousLastModified || cfg.startTime || new Date(0); + const maxTime = cfg.endTime || Date.MaxDate(); + let whereCondition; - if (previousLastModified === cfg.startTime || new Date(0)){ + if(snapshot.previousId) { + whereCondition = [ + pollingField = previousLastModified, + Id > snapshot.previousId + ]; + } else if (previousLastModified === cfg.startTime || new Date(0)){ whereCondition = [ - lastModified >= previousLastModified, - lastModified <= maxTime + pollingField >= previousLastModified, + pollingField <= maxTime ]; } else { whereCondition = [ - lastModified > previousLastModified, - lastModified <= maxTime + pollingField > previousLastModified, + pollingField <= maxTime ]; } const pageOfResults = GetPageOfResults({ - orderBy: Time ascending + orderBy: [Time ascending, Primary Key Ascending] where: whereCondition, - top: sizeOfPollingPage, - skip: snapshot.pageNumber * sizeOfPollingPage + top: cfg.sizeOfPollingPage }); - pageOfResults.forEach(result => { - emitData(result); - }; - snapshot.pageNumber++; - hasMorePages = pageOfResults.length == pageSize; - if(pageOfResults.length > 0) { - lastSeenTime = pageOfResults[pageOfResults.length - 1].lastModified; - } - emitSnapshot(snapshot); - if(singlePagePerInterval && hasMorePages) { - return; - } - } while (hasMorePages) - delete snapshot.pageNumber; - snapshot.previousLastModified = lastSeenTime; - emitSnapshot(snapshot); + + const hasMorePages = pageOfResults.length == cfg.sizeOfPollinPage; + + if(!hasMorePages) { + attemptMorePages = attemptMorePages && !snapshot.previousId; + pageOfResults.forEach(result => { + emitData(result); + }; + if(pageOfResults.length > 0) { + snapshot.previousLastModified = pageOfResults[pageOfResults.length - 1][pollingField]]; + delete snapshot.previousId; + emitSnapshot(snapshot); + } + } else { + const lastResult = pageOfResults.pop(); + pageOfResults.forEach(result => { + emitData(result); + }; + const secondLastResult = pageOfResults[pageOfResults.length - 1]; + snapshot.previousLastModified = secondLastResult[pollingField]; + if(lastResult[pollingField] !== secondLastResult[pollingField]) { + delete snapshot.previousId; + } else { + snapshot.previousId = secondLastResult.id; + } + emitSnapshot(snapshot); + } + } } +``` ##### Output Data - Each object emitted individually. -##### Gotcha’s to lookout for - -- If `previousLastModified` is set to `lastSeenTime` and we have `lastModified >= previousLastModified` then each execution will include records from previous execution. But if at the first execution `previousLastModified` could be equal `cfg.startTime` and we have `lastModified > previousLastModified` then we will lose objects whose last modified date is equal to the `cfg.startTime`. This is why we compare `previousLastModified` and `cfg.startTime || new Date(0)` and if they are equal, use condition `lastModified >= previousLastModified,` else: `lastModified > previousLastModified,` -- We use `lastModified <= maxTime` as it is more understandable for user. -- We have `Single Page per Interval` default to yes because it is slightly safer. -- TODO - ### Webhooks *This action has not been fully standardized.* Receives data pushed to the iPaas from an external system. +### Get Recently Deleted Objects Polling +##### Example Use Case +I want to learn about contacts in my CRM that are deleted so that I can propagate those deletes. + +##### Config Fields + +Same as `Get New and Updated Objects Polling`. +##### Input Metadata + +N/A + +##### Pseudo-Code + +Same as `Get New and Updated Objects Polling`. +##### Output Data + +- Each object emitted individually. + +### Event Subscription + +*This action has not been fully standardized.* + +The platform must have a part that is actively awake and is able to receive events based on some protocol. Examples: +* Salesforce Event Bus +* AMQP component +* Socket component +* JMX component +* CometD protocol +* Long polling + ### Bulk Extract Useful for: @@ -665,3 +757,4 @@ Useful for: - Systems that do no track last_modified - Systems that don’t support filtering by timestamp range - Systems which have dedicated bulk export functionality +- Providing a way to track object deletions From ddc8026db9a8587fc6a4d8d09439309d16e39049 Mon Sep 17 00:00:00 2001 From: Jacob Horbulyk Date: Fri, 4 Jun 2021 13:42:01 +0200 Subject: [PATCH 04/10] Add Section to Standardize Attachments (#8) * Add content for attachments. --- .../StandardizedActionsAndTriggers.md | 116 +++++++++++++++--- 1 file changed, 96 insertions(+), 20 deletions(-) diff --git a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md index 881d789..f16c0ee 100644 --- a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md +++ b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md @@ -1,8 +1,8 @@ # Descriptions of standardized actions or triggers -**Version Publish Date:** 18.02.2020 +**Version Publish Date:** 25.05.2021 -**Semantic Version of Document:** 2.3.1 +**Semantic Version of Document:** 2.4.0 ## Table of Contents @@ -27,6 +27,9 @@ * [Get Recently Deleted Objects Polling](#get-recently-deleted-objects-polling) * [Event Subscription](#event-subscription) * [Bulk Extract](#bulk-extract) +- [Attachments](#attachments) + * [List of Attachment Meta-Information Fields](#list-of-attachment-meta-information-fields) + * [Attachment Examples](#attachment-examples) It is important to define common rules on how an adapter responds to changes and performs actions on generic domain objects. If adapters follow @@ -118,7 +121,7 @@ I have a contact who works for a company. I have an ID or other distinguishing - Object Type (dropdown) - Allow ID to be omitted (dropdown/checkbox: yes/no); when selected, the ID field becomes optional, otherwise it is a required field -- Allow zero results (dropdown/checkbox: yes/no); When selected, if zero results are returned, the empty object `{}` is emitted, otherwise typically an error would be thrown. +- Allow zero results (dropdown/checkbox: yes/no); When selected, if zero results are returned, the empty object `{}` is emitted, otherwise typically an error would be thrown. - Wait for object to exist (dropdown/checkbox: yes/no); When selected, if no results are found, apply rebounds and wait until the object exits. - Linked objects to populate (optional, multi-select dropdown). Select which linked objects to fetch if supported by the API. @@ -363,7 +366,7 @@ I salesperson is responsible for 0 to N accounts (N being reasonably small). I throw new NotFoundError(); } emitRebound({}); - return; + return; } return { key: itemCriteria, @@ -450,8 +453,8 @@ See above. throw new NotFoundError(); } emitRebound(); - return; - } + return; + } const object1Id = matchingObjects1[0].id; const matchingObjects2 = lookupObjectByCriteria(obj2.type, obj2.uniqueCriteria); @@ -462,7 +465,7 @@ See above. throw new NotFoundError(); } emitRebound(); - return; + return; } const object2Id = matchingObjects2[0].id; @@ -658,8 +661,8 @@ In order for the bellow polling algorithm to work, all of the following must be let whereCondition; if(snapshot.previousId) { - whereCondition = [ - pollingField = previousLastModified, + whereCondition = [ + pollingField = previousLastModified, Id > snapshot.previousId ]; } else if (previousLastModified === cfg.startTime || new Date(0)){ @@ -679,33 +682,33 @@ In order for the bellow polling algorithm to work, all of the following must be where: whereCondition, top: cfg.sizeOfPollingPage }); - + const hasMorePages = pageOfResults.length == cfg.sizeOfPollinPage; - + if(!hasMorePages) { - attemptMorePages = attemptMorePages && !snapshot.previousId; + attemptMorePages = attemptMorePages && !snapshot.previousId; pageOfResults.forEach(result => { emitData(result); - }; + }; if(pageOfResults.length > 0) { snapshot.previousLastModified = pageOfResults[pageOfResults.length - 1][pollingField]]; delete snapshot.previousId; - emitSnapshot(snapshot); - } + emitSnapshot(snapshot); + } } else { const lastResult = pageOfResults.pop(); pageOfResults.forEach(result => { emitData(result); - }; + }; const secondLastResult = pageOfResults[pageOfResults.length - 1]; - snapshot.previousLastModified = secondLastResult[pollingField]; + snapshot.previousLastModified = secondLastResult[pollingField]; if(lastResult[pollingField] !== secondLastResult[pollingField]) { - delete snapshot.previousId; + delete snapshot.previousId; } else { snapshot.previousId = secondLastResult.id; } - emitSnapshot(snapshot); - } + emitSnapshot(snapshot); + } } } ``` @@ -758,3 +761,76 @@ Useful for: - Systems that don’t support filtering by timestamp range - Systems which have dedicated bulk export functionality - Providing a way to track object deletions + +## Attachments + +This section describes what meta-information for attachments should be communicated as part of the message body and how this meta-information should be included. + +There are the following cases where a component would produce an attachment: +* **Binary Field Case:** There is some binary/non-text data that is stored as part of an entity (e.g. Photo of a contact in a CRM) +* **File System Case:** The component is interacting with a 3rd party system that could be described as being a "drive" that exposes a file system (e.g. SFTP, Google Drive, etc.) +* **File Generation Case:** The component creates an attachment of a specified type based on the inputs to the step (e.g XML or CSV component creates XML/CSV files based on incoming JSON data) + +For the **Binary Field Case**, we would expect the attachment information to appear as an inline object that is appropriately placed within the message body object. Otherwise, for the **File System Case** and the **File Generation Case** the file attachment meta-information can be an object that occupies the entire message body. + +### List of Attachment Meta-Information Fields +* `attachmentUrl` (Expected for all three cases): URL to a steward or maester object containing the (potentially) binary data of the attachment +* `size` (Expected for all three cases): Size in bytes of the attachment +* `name` (Expected only for the **File System Case**): Name of the file in the "drive" +* `path` (Expected only for the **File System Case**): Path of the file within the "drive" (includes filename) +* `directory` (Expected only for the **File System Case**): Path of the file within the "drive" (excludes filename) +* `type` (Optional for the **File System Case**, Expected for the **File System Case** and **File Generation Case**): Filename extension suffix (e.g. `.csv`, `.jpg`) of the file +* `modifyTime` (Expected for **File System Case**, optional for **Binary Field Case**): Time when the file was created +* `attachmentCreationTime` (Expected for all three cases): Time when the attachment was created +* `attachmentExpiryTime` (Expected for all three cases): Time when the attachment will be deleted +* `contentType` (Expected for only the **Binary Field Case** and **File Generation Case**): [Value that would normally appear in a `Content-Type` HTTP header field (e.g. `application/xml`, `image/jpeg`)](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types) + + +### Attachment Examples +#### Binary Field Case +For a contact photo +```json +{ + "contact": { + "firstName": "Fred", + "lastName": "Smith", + "profilePhoto": { + "attachmentUrl": "http://steward-service.platform.svc.cluster.local:8200/files/8a10d010-10f8-4d1d-88ff-7458f66d574d", + "size": 126976, + "modifyTime": "2021-05-01T12:00:00.000Z", + "attachmentCreationTime": "2021-05-25T12:00:00.000Z", + "attachmentExpiryTime": "2021-05-27T12:00:00.000Z", + "contentType": "image/jpeg" + }, + ... + } +} +``` + +#### File System Case +```json +{ + "attachmentUrl": "http://steward-service.platform.svc.cluster.local:8200/files/8a10d010-10f8-4d1d-88ff-7458f66d574d", + "size": 126976, + "name": "profilePhoto.jpg", + "path": "/home/someuser/Documents/profilePhoto.jpg", + "directory": "/home/someuser/Documents", + "type": ".jpg", + "modifyTime": "2021-05-01T12:00:00.000Z", + "attachmentCreationTime": "2021-05-25T12:00:00.000Z", + "attachmentExpiryTime": "2021-05-27T12:00:00.000Z" +} +``` + +#### File Generation Case +```json +{ + "attachmentUrl": "http://steward-service.platform.svc.cluster.local:8200/files/8a10d010-10f8-4d1d-88ff-7458f66d574d", + "size": 126976, + "type": ".xml", + "modifyTime": "2021-05-01T12:00:00.000Z", + "attachmentCreationTime": "2021-05-25T12:00:00.000Z", + "attachmentExpiryTime": "2021-05-27T12:00:00.000Z", + "contentType": "application/xml" +} +``` From 64955c26cb4b80be89d9865811cc790def3ef712 Mon Sep 17 00:00:00 2001 From: Jacob Horbulyk Date: Fri, 4 Jun 2021 13:43:56 +0200 Subject: [PATCH 05/10] Clean Up lookup objects action. (#9) * Clean-up lookup objects action. --- .../StandardizedActionsAndTriggers.md | 44 ++++++++----------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md index f16c0ee..15159ee 100644 --- a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md +++ b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md @@ -1,8 +1,8 @@ # Descriptions of standardized actions or triggers -**Version Publish Date:** 25.05.2021 +**Version Publish Date:** 26.05.2021 -**Semantic Version of Document:** 2.4.0 +**Semantic Version of Document:** 2.5.0 ## Table of Contents @@ -208,23 +208,19 @@ I want to search my CRM for data based on some criteria. ##### Config Fields -- Object Type (dropdown) -- Behavior (dropdown: Fetch all, Fetch Page, Emit Individually) -- Number of search terms (text field: integer >= 0) (iteration 2) (0 indicates return all items) -- Linked objects to populate (optional, multi-select dropdown). Select which linked objects to fetch if supported by the API. +- Object Type (dropdown, required) +- Behavior (dropdown: Fetch All, Fetch Page, Emit Individually, required) +- Linked objects to populate (multi-select dropdown, optional): Select which linked objects to fetch if supported by the API. ##### Input Metadata -- Page size: optional positive integer that defaults to 100 (only if fetch page mode) -- page number: required non-negative integer that is 0 based (only if fetch page mode) -- order: optional array of fieldname + sort direction pairs (only if fetch page mode) -- max result size: optional positive integer that defaults to 1000 (only if fetch all mode) -- For each search term: +- Page Size (non-negative integer, optional: defaults to page size used by API): (only if fetch page mode) A value of `0` indicates that the `results` will be an empty array with only `totalCountOfMatchingResults` populated. +- Page Number (non-negative 0 based integer, optional: default to 0): (only if fetch page mode) +- Order (Array of fieldname + sort direction pairs, optional: default to empty array): (only if fetch page mode) +- Search Criteria: (optional: default to empty array). Search terms are to be combined with the *AND* operator. For each search term: - fieldName - fieldValue - - condition (equal, not equal, >=, <=, >, <, like (if supported), possibly more in the future) -- For each search term - 1: (iteration 2) - - criteriaLink (and/or) + - condition (equal, not equal, >=, <=, >, <, like (if supported), whatever else is suppored by the API) ##### Pseudo-Code @@ -232,9 +228,6 @@ I want to search my CRM for data based on some criteria. switch(mode) { case 'fetchAll': const results = GetObjectsByCriteria(criteria, linkedObjectsToPopulate); - if(results.length >= maxResultSize) { - throw new Error('Too many results'); - } emitData({results: results}); break; case 'emitIndividually': @@ -244,25 +237,26 @@ I want to search my CRM for data based on some criteria. } break; case 'fetchPage': - const results = GetObjectsByCritieria(criteria, top: pageSize, skip: pageSize * pageNumber, orderBy: orderByTerms, linkedObjectsToPopulate); - emitData({results: results}); + const {results, totalCountOfMatchingResults} = GetObjectsByCritieria(criteria, top: pageSize, skip: pageSize * pageNumber, orderBy: orderByTerms, linkedObjectsToPopulate); + emitData({results, totalCountOfMatchingResults}); break; } } ##### Output Data - -- An object, with key `results` that has an array as its value. +- For **Fetch Page** mode:An object with + - key `results` that has an array as its value + - key `totalCountOfMatchingResults` which contains the total number of results (not just on the page) which match the search criteria +- For **Fetch All** mode: An object, with key `results` that has an array as its value. +- For **Emit Individually** mode: Each object should fill the entire message. ##### Gotcha’s to lookout for - Make sure to Url Encode field values appearing in HTTP urls +- The page size requested for the component may be larger that the page size supported by the API. In this case, the component must fetch multiple pages from the API to combine into a result. ##### Not Handled - -- Order of operations in multiple terms -- How to get total number of matching objects -- How to handle variable number of search terms (perhaps integrator mode?) +- Using the *OR* operator to combine search terms ### Delete Object ##### Example Use Case From dc01f907db98ec08f7b53e19d2fa7de71a26dfe7 Mon Sep 17 00:00:00 2001 From: Jacob Horbulyk Date: Fri, 4 Jun 2021 16:51:19 +0200 Subject: [PATCH 06/10] Fix example. (#10) --- .../StandardizedActionsAndTriggers.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md index 15159ee..f7122c0 100644 --- a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md +++ b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md @@ -1,8 +1,8 @@ # Descriptions of standardized actions or triggers -**Version Publish Date:** 26.05.2021 +**Version Publish Date:** 04.06.2021 -**Semantic Version of Document:** 2.5.0 +**Semantic Version of Document:** 2.5.1 ## Table of Contents @@ -822,7 +822,6 @@ For a contact photo "attachmentUrl": "http://steward-service.platform.svc.cluster.local:8200/files/8a10d010-10f8-4d1d-88ff-7458f66d574d", "size": 126976, "type": ".xml", - "modifyTime": "2021-05-01T12:00:00.000Z", "attachmentCreationTime": "2021-05-25T12:00:00.000Z", "attachmentExpiryTime": "2021-05-27T12:00:00.000Z", "contentType": "application/xml" From a2516a0876fbcd412a83e346c4b426c4e4a00537 Mon Sep 17 00:00:00 2001 From: Jacob Horbulyk Date: Wed, 28 Jul 2021 11:57:55 +0200 Subject: [PATCH 07/10] Standardize "Make Raw Request". (#12) * Standardize "Make Raw Request". * Extend error handling of raw request. * Rename raw request to raw HTTP request. --- .../StandardizedActionsAndTriggers.md | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md index f7122c0..b4be657 100644 --- a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md +++ b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md @@ -1,8 +1,8 @@ # Descriptions of standardized actions or triggers -**Version Publish Date:** 04.06.2021 +**Version Publish Date:** 12.07.2021 -**Semantic Version of Document:** 2.5.1 +**Semantic Version of Document:** 2.6.0 ## Table of Contents @@ -11,7 +11,7 @@ * [Lookup Object (at most 1)](#lookup-object-at-most-1) * [Lookup Objects (Plural)](#lookup-objects-plural) * [Delete Object](#delete-object) - * [Make RAW Request](#make-raw-request) + * [Make Raw HTTP Request](#make-raw-http-request) * [Lookup Set Of Objects By Unique Criteria](#lookup-set-of-objects-by-unique-criteria) * [Update Object](#update-object) * [Create Object](#create-object) @@ -317,18 +317,34 @@ I know the ID of a customer that I want to delete. } } -### Make RAW Request +### Make Raw HTTP Request +A simple action to allow integrators to assemble requests to be sent to the system. The component should expose the parts that vary in a typical request. The component should handle authentication and error reporting. It can optionally handle paging. In some cases, it may make sense to allow the option to make a series of sequential array requests though this later behavior is not yet standardized. -*This action has not been fully standardized.* +##### Example Use Case +I'm a technically advanced user who wants to interact with a system in a way not permissible by the existing component actions but would like some simplification relative to using the REST component. -A simple action to allow integrators to assemble requests to be sent to the system. The component should expose the parts that vary in a typical request. The component should handle authentication and error reporting. +##### Config Fields +* Error Tolerance (dropdown, required): Determines behavior for when an erroneous HTTP code is received. Options are as follows: + * **No Errors Tolerated**: Any HTTP status code >= 400 should result in an error being thrown + * **Only Not Found Errors Tolerated**: HTTP status codes of 404, 410 or similar should result in a message being produced with the status code and the HTTP reponse. All other error codes should result in an error being thrown. + * **None**: Regardless of the HTTP error code, the component should produce an outbound message with the status code and the HTTP reponse. + * **Manual**: A range of error codes to throw errors on can be configured via the message input. -Additional Options to Consider: -* Consider that it may make sense to turn off error reporting & to return things like the HTTP status code & headers -* Consider that it may make sense to allow the option to make a series of sequential array requests. +##### Input Metadata +* Url (string, required): Path of the resource relative to the URL base. +* Method (string enum (Enum options are system specific.), required): HTTP Verb for the request. +* Request Body (object, optional): Body of the request to send -##### Example Use Case -I'm a technically advanced user who wants to interact with a system in a way not permissible by the existing component actions but would like some simplification relative to using the REST component. +If **Error Tolerance** is **Manual**: + * HTTP Codes to throw errors (array of error ranges, optional default to `[]`): A double array with a list of ranges of HTTP response codes to throw errors upon receiving Use a syntax that matches [retry-axios](https://www.npmjs.com/package/retry-axios). Example: `[[400, 403], [405,599]]` - Throw errors on all errors apart from 404. + +*For some systems, it may make sense to also add the HTTP headers.* + +##### Output Data +* Status Code (integer, required): HTTP status code of the request +* Response Body (object, optional): JSON representation of the response body from the request + +*For some systems, it may make sense to also add the HTTP headers.* ### Lookup Set Of Objects By Unique Criteria Given an array of information where each item in the array uniquely describes exactly one object. It can be assumed that the array is short enough to reasonably fit the results in a single message. If any of the objects are not found then it indicates a logic problem in the integration. From 920ef752bc1efeff2d17aab80600d3cacc2741ab Mon Sep 17 00:00:00 2001 From: Pavel Voropaiev Date: Thu, 13 Apr 2023 10:59:49 +0300 Subject: [PATCH 08/10] Delete object revitalize (#13) --- .../StandardizedActionsAndTriggers.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md index b4be657..fa74809 100644 --- a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md +++ b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md @@ -267,6 +267,10 @@ I know the ID of a customer that I want to delete. ##### Config Fields - Object Type (dropdown) +- Emit strategy when no object found (dropdown). Available options: + - “Emit nothing” + - “Emit an empty object {}” + - “Throw an error” ##### Input Metadata @@ -278,7 +282,7 @@ I know the ID of a customer that I want to delete. try { DeleteObjectById(id); // Usually DELETE verb } catch (NotFoundException e) { - emitData({}); + emitDataBasedOnEmitStrategySelected(); return; } emitData({id: id}); @@ -308,7 +312,7 @@ I know the ID of a customer that I want to delete. function deleteObjectByUniqueCriteria(uniqueCriteria) { const foundObjects = GetObjectsByCritieria(uniqueCriteria); // Usually GET verb if(foundObjects.length == 0) { - emitData({}); + emitDataBasedOnEmitStrategySelected(); } else if (foundObjects.length == 1) { DeleteObjectById(foundObjects[0].id); // Usually DELETE verb emitData({id: foundObjects[0].id}); From 7a3620e3f21a33f6bd35ce0e962c12761dfbdf09 Mon Sep 17 00:00:00 2001 From: Alex Motsak Date: Thu, 13 Apr 2023 12:12:41 +0300 Subject: [PATCH 09/10] Lookup Objects (Plural) revitalize (#14) --- .../StandardizedActionsAndTriggers.md | 64 ++++++++++++------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md index fa74809..45da9c6 100644 --- a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md +++ b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md @@ -210,37 +210,57 @@ I want to search my CRM for data based on some criteria. - Object Type (dropdown, required) - Behavior (dropdown: Fetch All, Fetch Page, Emit Individually, required) -- Linked objects to populate (multi-select dropdown, optional): Select which linked objects to fetch if supported by the API. +- Number of Search Terms (number [0-99], optional) +- Linked objects to populate (multi-select dropdown, optional): Select which linked objects to fetch if supported by the API. +- Enable Expert mode (checkbox, optional): If selected, `Number of Search Terms` will be ignored, instead of it inside input metadata will be oly one field - `Filter Expression` + +
Metadata example: + + ```json + { + "filterExpression": "((name = 'John') and (age between 20 and 30)) or (name = 'Julie')" + } + ``` +
##### Input Metadata - Page Size (non-negative integer, optional: defaults to page size used by API): (only if fetch page mode) A value of `0` indicates that the `results` will be an empty array with only `totalCountOfMatchingResults` populated. - Page Number (non-negative 0 based integer, optional: default to 0): (only if fetch page mode) -- Order (Array of fieldname + sort direction pairs, optional: default to empty array): (only if fetch page mode) -- Search Criteria: (optional: default to empty array). Search terms are to be combined with the *AND* operator. For each search term: - - fieldName - - fieldValue - - condition (equal, not equal, >=, <=, >, <, like (if supported), whatever else is suppored by the API) +- Order (Array of fieldname + sort direction pairs, optional: default to empty array) +- Groups of fields for each search term if `Number of Search Terms` more than 0 and `Enable Expert mode` not selected + - Field name (string, required): Object field name to filter + - Condition (string, required): Condition to compare selected field with value, depends from API + - Field value (string, optional) Value of selected field, may be empty in cases where condition like `CHANGED` of `EXIST` + - Logical operator (string, required): Appears only if there is more than one search term to combine multiple search terms +- If selected `Enable Expert mode` + - Filter Expression (string, required) custom string to filter Objects, for advanced users ##### Pseudo-Code function lookupObjects(criteria) { - switch(mode) { - case 'fetchAll': - const results = GetObjectsByCriteria(criteria, linkedObjectsToPopulate); - emitData({results: results}); - break; - case 'emitIndividually': - const results = GetObjectsByCriteria(criteria, linkedObjectsToPopulate); - results.forEach(result => { - emitData(result); - } - break; - case 'fetchPage': - const {results, totalCountOfMatchingResults} = GetObjectsByCritieria(criteria, top: pageSize, skip: pageSize * pageNumber, orderBy: orderByTerms, linkedObjectsToPopulate); - emitData({results, totalCountOfMatchingResults}); - break; - } + var proceed = true; + vat top = pageSize || 100; + var skip = top * (pageNumber || 0); + var combinedResults = []; + do { + const { results, totalCountOfMatchingResults } = GetObjectsByCriteria(criteria, linkedObjectsToPopulate, top, skip, orderByTerms) + switch(mode) { + case 'fetchAll': + combinedResults.push(...results); + break; + case 'emitIndividually': + results.forEach(result => { emitData(result) }); + break; + case 'fetchPage': + emitData({results, totalCountOfMatchingResults}); + proceed = false; + break; + } + skip += top; + if (results.length == 0) proceed = false; + } while (proceed) + if (mode == 'fetchAll') emitData({results: combinedResults}); } ##### Output Data From ea930f0df2e20b40dade5297438b53278583d937 Mon Sep 17 00:00:00 2001 From: Alex Motsak Date: Fri, 14 Apr 2023 14:15:01 +0300 Subject: [PATCH 10/10] Get New and Updated Objects Polling revitalize --- .../StandardizedActionsAndTriggers.md | 97 ++++++------------- 1 file changed, 29 insertions(+), 68 deletions(-) diff --git a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md index 45da9c6..8f893a4 100644 --- a/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md +++ b/Adapters/AdapterBehaviorStandardization/StandardizedActionsAndTriggers.md @@ -654,9 +654,9 @@ I want to learn about changes to contacts in my CRM when they happen. - Object Type (dropdown) - Start Time (string, optional): Indicates the beginning time to start polling from (defaults to the beginning of time) -- End Time (string, optional): If provided, don’t fetch records modified after this time (defaults to never) -- Size of Polling Page (optional; positive integer) Indicates the size of pages to be fetched. Defaults to 1000. -- Single Page per Interval (dropdown/checkbox: yes/no; default yes) Indicates that if the number of changed records exceeds the maximum number of results in a page, instead of fetching the next page immediately, wait until the next flow start to fetch the next page. +- End Time (string, optional): If provided, don’t fetch records modified after this time (defaults to now, where ‘now’ is each trigger execution’s timestamp) +- Behavior (dropdown: Fetch Page, Emit Individually, required) +- Size of Polling Page (optional; positive integer) if size of a page > maximum allowed by API then page size = maximum allowed by API otherwise - a user-specified value - Time stamp field to poll on (dropdown: created or modified). Indicates just new items or new and modified items. ##### Input Metadata @@ -664,92 +664,53 @@ I want to learn about changes to contacts in my CRM when they happen. N/A ##### Gotcha’s to lookout for - -- If `previousLastModified` is set to `lastSeenTime` and we have `lastModified >= previousLastModified` then each execution will include records from previous execution. But if at the first execution `previousLastModified` could be equal `cfg.startTime` and we have `lastModified > previousLastModified` then we will lose objects whose last modified date is equal to the `cfg.startTime`. This is why we compare `previousLastModified` and `cfg.startTime || new Date(0)` and if they are equal, use condition `lastModified >= previousLastModified,` else: `lastModified > previousLastModified,` -- We use `lastModified <= maxTime` as it is more understandable for user. -- We have `Single Page per Interval` default to yes because it is slightly safer. - We need to be careful about more than a page worth of records having the same timestamp. - We need to be careful about the last record on one page having the same timestamp as the first record on the next page - We need to be careful about records on page N being modified before reading page N+1 (thus causing records to be skipped as they move from page N+1 to page N). ##### Assumptions About Server Behavior: In order for the bellow polling algorithm to work, all of the following must be true about the way the server behaves: -* It is possible to order results by last modified and then by primary key. * If record A has a timestamp of X and appears within a search but not record B, then if record B appears in a later search than record B MUST have a timestamp that is later (i.e. Not the same or earlier) than record A. ##### Pseudo-Code - -**High level steps:** -1. Retrieve a page of data. -2. If the size of the page is less than the max page size, emit the timestamp of the last record on the page. -3. Compare the timestamps of the last and second last items on the page. If they are different, store the timestamp of the second last record of the page and don't emit the last record. -4. If the timestamps are the same, then store that timestamp and the primary key of the last item. On the next iteration fetch page where the timestamps are equal and primary key is larger than last seen item. - ``` function getObjectsPolling(cfg, snapshot) { - const pollingField = cfg.timeStampFieldToPollOn; - let attemptMorePages = !cfg.singlePagePerInterval; - do { - const previousLastModified = snapshot.previousLastModified || cfg.startTime || new Date(0); - const maxTime = cfg.endTime || Date.MaxDate(); - - let whereCondition; - if(snapshot.previousId) { - whereCondition = [ - pollingField = previousLastModified, - Id > snapshot.previousId - ]; - } else if (previousLastModified === cfg.startTime || new Date(0)){ - whereCondition = [ - pollingField >= previousLastModified, - pollingField <= maxTime - ]; - } else { - whereCondition = [ - pollingField > previousLastModified, - pollingField <= maxTime - ]; - } + const currentTime = new Date(); + const { startTime, endTime, pageSize, emitBehavior, objectType, pollConfig } = cfg; + const from = snapshot?.nextStartTime || startTime || 0; + const to = endTime || currentTime; + const nextStartTime = currentTime; - const pageOfResults = GetPageOfResults({ - orderBy: [Time ascending, Primary Key Ascending] - where: whereCondition, - top: cfg.sizeOfPollingPage - }); + let proceed = true; + let pageNumber = 1; - const hasMorePages = pageOfResults.length == cfg.sizeOfPollinPage; + do { + const results = getPageOfResults({objectType, pollConfig, pageSize, pageNumber, from, to}); + pageNumber++; - if(!hasMorePages) { - attemptMorePages = attemptMorePages && !snapshot.previousId; - pageOfResults.forEach(result => { - emitData(result); - }; - if(pageOfResults.length > 0) { - snapshot.previousLastModified = pageOfResults[pageOfResults.length - 1][pollingField]]; - delete snapshot.previousId; - emitSnapshot(snapshot); + if (results.length !== 0) { + if (results.length < Number(pageSize)) { + proceed = false; + emitSnapshot({ nextStartTime }); } - } else { - const lastResult = pageOfResults.pop(); - pageOfResults.forEach(result => { - emitData(result); - }; - const secondLastResult = pageOfResults[pageOfResults.length - 1]; - snapshot.previousLastModified = secondLastResult[pollingField]; - if(lastResult[pollingField] !== secondLastResult[pollingField]) { - delete snapshot.previousId; - } else { - snapshot.previousId = secondLastResult.id; + if (emitBehavior === 'emitPage') { + emitData({results}); + } else if (emitBehavior === 'emitIndividually') { + for (const record of results) { + emitData(record); + } } - emitSnapshot(snapshot); + } else { + await this.emit('snapshot', { nextStartTime }); + proceed = false; } - } - } + } while (proceed); ``` ##### Output Data -- Each object emitted individually. +- For **Fetch Page** mode: An object, with key `results` that has an array as its value. +- For **Emit Individually** mode: Each object should fill the entire message. ### Webhooks