Derived data rule language

An example rule

 
{
    "name":"my DD rule", // required; unique in tenant
    "ruleType": "derived", // required
    "isGloballyVisible": false, // optional;  default=false
    "createdAt": "2020-12-11T12:01:48Z",  // generated; readonly
    "updatedBy": "fe820f40-35ae-4c92-a09b-62c35bbec155", // generated; readonly
    "tenantId": "1aaeef92-7eca-47ea-9545-d9b3a6c27416", // optional; default=global
    "ruleId": "40950af9-1f6b-46c2-ac32-7cbec25464b1",  // generated; readonly
    "version": 1.0, // generated; readonly
    "updatedAt": "2020-12-11T12:01:48Z", // generated; readonly
    "status": { // generated; readonly
        "code": "live",
        "details": "Version 1.0 of this rule is live and processing"
    },
    "fieldsSetByRule":[   // generated; readonly
        {
            "fieldName":"",
            "fieldType":""
        },
        {
            "fieldName":"",
            "fieldType":""
        },
        ...
    ], 
    "objectTypeSetByRule": "Issue",   // generated; readonly
    "instructions":{ // required
     "enableCollapse": true, // optional; default=false
     "throttling": { // optional 
            "window": "15.minutes",
            "enabled": "false"
        },
     "ifEvent": { // required
           "AND": [ // root-level
                    {
                        "EQUALS": [
                            {
                                "SELECT": "",
                                "TYPE": "String"
                            },
                            {
                                "VALUE": "",
                                "TYPE": "String"
                            }
                        ]
                    },
                    {
                        "OR": [
                            {
                                "EXISTS": [
                                    {
                                        "SELECT": "",
                                        "TYPE": "String"
                                    }
                                ]
                            },
                            {
                                "EXISTS": [
                                    {
                                        "SELECT": "",
                                        "TYPE": "String"
                                    }
                                ]
                            }
                        ]
                    }
                ]
        },
    "sendObjectUpsertEvents": { // required
            "objects": {// required
                "id": { // required
                    "select": ""
                },
                "type": { // required
                    "select": ""
                }
            },
            "map": [ // required               
                {                 
                    "sequence": "1",
                    "expression":{ // required
                        "QUERY": {// optional
			    "hint": "objectwarehouse-adhoc", // required
                            "advancedFilter": {  // optional
                                "AND": [
                                    {
                                        "EQUALS": [
                                            {
                                                "FIELD": "",  // optional
                                                "TYPE": "" // required
                                            },
                                            {
                                                "TYPE": "",
                                                "SELECT": ""
                                            }
                                        ]
                                    },
                                    {
                                        "EQUALS": [
                                            {
                                                "FIELD": "",
                                                "TYPE": "String"
                                            },
                                            {
                                                "VALUE": "",  // optional
                                                "TYPE": "String"
                                            }
                                        ]
                                    }
                                ]
                            },
                            "reduce": { // optional
                                "expression": { // required
                                    "fieldPath": "",
                                    "operator": "SUM"
                                },
                                "output_local": { // required
                                    "resultAlias": ""
                                }
                            }
                          }
                    }
                },
                {
                    "sequence": 2, // required
                    "nestedUpserts": { // optional
                        "nestedPath": "", // required
                        "objects": {
                            "localIdentifiers": {
                                "SELECT": "localIdentifiers"
                            }
                        },
                        "map": [ // required
                            {
                                "sequence": 1,
                                "expression": {
                                    "QUERY": {
                                        "hint": "orderflow",
                                        "output_local": {
                                            "fields": [
                                            ...
                                            ],
                                            "resultAlias": ""
                                        }
                                    }
                                }
                            },
                            {
                                "sequence": 2,
                                "output": {
                                    "resultFieldPath": "",
                                    "resultType": "String"
                                },
                                "if": {// optional
                                    "AND": [
                                             {
                                                "EQUALS": [
                                                   {
                                                     "SELECT_LOCAL": "",
                                                     "TYPE": "String"
                                                   },
                                                   {
                                                      "VALUE": "value",
                                                      "TYPE": "String"
                                                   }
                                                ]
                                             },
                                             {
                                                "GREATER_THAN": [
                                                 {
                                                    "SELECT_LOCAL": "",
                                                    "TYPE": "float"
                                                 },
                                                 {
                                                    "VALUE": 0,
                                                    "TYPE": "float"
                                                 }
                                              ]
                                           }
                                      ]
                                 },
                                "expression": {
                                    "value": "value",
                                    "type": "String"
                                },
                                "breakOnMatch": true // optional
                            },
                            {
                                "sequence": 3,
                                "output": {
                                    "resultFieldPath": "",
                                    "resultType": "String"
                                },
                                "expression": {
                                    "minus": [ // optional
                                        {
                                            "SELECT_LOCAL": "",
                                            "TYPE": "Date"
                                        },
                                        {
                                            "SELECT_LOCAL": "",
                                            "TYPE": "float"
                                        }
                                    ]
                                }
                            }
                        ]
                    }
                }
          ]
    }   
}    

Where:

isGloballyVisible

Optional Boolean value that indicates whether this rule filter event by tenantId. Default value is false.

fieldsSetByRule

Array of JSON objects and each object contains the fieldName and fieldType.

Generated by the suite, the array is collected from output in the top level of each map section.

objectTypeSetByRule

A string that indicates the object type that the derived data rule upserts. Generated by the suite.

Rule instructions documentation

"instructions: {

enableCollapse

Optional Boolean value that specifies whether this rule needs to merge the related events with same business object ID in the same batch into one output event, which also can improve performance. Default value is false.

throttling

Optional time window that specifies the interval to trigger derived data rule. By default, Derived Data rule is triggered by event. But after you enable the time window, the rule will be triggered by interval, and the events in different intervals will be processed in batches, which also can improve performance.

    "throttling": { 

window

Required string value that specifies timer. The format must be integer.timeUnit, for example "1.minute","15.minutes","2.m". Time unit must be in second, minute, hour, day or "s","m","h","d". For example, "window" : "15.minutes" means that this rule is triggered at :15, :30, :45 and :00 every hour.

 	    "window":

enabled

Required Boolean that specifies whether to enable throttling. Default value is false.

	    "enabled":
}

ifEvent

Required initial event that specifies filter conditions. The operator must be in the list of EXISTS, GREATER_THAN, GREATER_EQUALS, LESS_THAN, LESS_EQUALS, EQUALS, UNEQUALS, OR, AND, POINTS_IN_CIRCLES, REFERENCED_BY.

Required string that specifies one from SELECT and VALUE in every operand and must indicate TYPE for the field you want to select or value you specified.

TYPE, SELECT, and VALUE are case-insensitive.

SELECT allowing field and field path.

	"ifEvent": {
		operator:[
			{
				operator: [ operand, operand ]
			},
			{
				operator: [ operand, operand ]
			}
		]
	}					

sendObjectUpsertEvents

Required object for processing and sending the result.

	"sendObjectUpsertEvents": {

objects

Required object that specifies the fields that you want to get from the initial event.

Can specify only fields "id" and "type" in sendObjectUpsertEvents.object.

	    "objects": {

id

Required object that specifies the event ID.

  	        "id": {

select

Required string that specifies the field path of the event, can be separated by a period (.). For example, "select": "a.b.c"

	           	"select": ""
			}

type

Required string that specifies the event type.

  	        "type": {

select

Required string that specifies the field path of the event.

	           	 "select": ""
		         }
		     }

map

Required array that specifies how to process events to get derived data. It also supports nestedUpserts to handle the event with multiple line-level events.

Allowing these fields in every map element.

Property	Type	Required
sequence	string	Yes
expression	string	Yes
nestedUpserts	object	No
if	object	No
output	object	No
breakOnMatch	string	No

	    "map":[
		   {

sequence

Required string that specifies a number for every element of the map array. Statement blocks are executed in ascending order based on the specified sequence number.

		   "sequence": ""

output

Optional object that specifies where you want to output result, allows field path format, such as a.b.c or array a[name=globalIdentifiersTest].b

                               "output": {                                    

resultFieldPath

Required string that supports output object or array. Object needs to specify the path, such as a.b.c. Array needs to specify the array path, such as a[name=nameValue].b. When the output is an array, an element in the array is located based on the name, so the name must be unique in the array.

                        "resultFieldPath": "a.b.c", //object
                        "resultFieldPath":"globalIdentifiers[name=globalIdentifiersTest].value", //array

resultType

Required string that specifies output data type. The value must be in int, long, double, float, String, Date, Boolean, Object, Array.

                        "resultType":"string"

expression

Required object expression that allows QUERY , CONCATENATE, math operator and specifies VALUE directly.

The math operator in the root-level needs to be in the list of MULTIPLY, DIVIDE, ADD, SUBTRACT, MOD, MIN, MAX, ABS, SQRT, ROUND.

The operator in the second level supports more types, including MULTIPLY, DIVIDE, ADD, SUBTRACT, MOD, MIN, MAX, ABS, SQRT, ROUND, SELECT, SELECT_LOCAL, VALUE.

Note: The SELECT_LOCAL operator is used to query the fields that are saved in a temporary file. For example, the fields are saved in cache after the expression.QUERY.output_local is executed.

                   "expression": {

CONCATENATE

Optional CONCATENATE array that can support one or more elements. Each element in the array supports select, select_local,value. CONCATENATE concatenates the calculation results of each element in the order of the array as the calculation result of the expression.

"expression":{
                        "CONCATENATE":[ // optional
                            {
                                "SELECT":"timestampEventRecieved",
                                "TYPE":"String",
                                "ORDER": 1
                            },
                            {
                                "SELECT_LOCAL":"lineCountsByShipmentStatus.FULLY_SHIPPED",
                                "TYPE":"String",
                                "ORDER": 2
                            },
                            {
                                "VALUE":"ddRuleTestId12345666",
                                "TYPE":"String",
                                "ORDER": 3
                            }
                        ]
                    }

QUERY

Optional object query from the specified index.

                       "QUERY": {				        

hint

Required string that specifies the index that you want to query.

		          "hint":""

advancedFilter

Optional object that specifies query conditions to filter event from the specified index. The definition of advancedFilter here is same as the advancedFilter GraphQL query. The operator must be in the list of CONTAINS, CONTAINS_ANY, EXISTS, GREATER_THAN, GREATER_EQUALS, LESS_THAN, LESS_EQUALS, EQUALS, EQUALS_ANY, UNEQUALS, NOT, OR, AND, POINTS_IN_CIRCLES, REFERENCED_BY.

Required string that specifies one from FIELD, SELECT, and VALUE in every operand and must indicate TYPE for the field that you want to select or the value that you specified.

                           "advancedFilter": {
                                  operator:[
			           {
				     operator: [ operand, operand ]
			           },
			           {
				    operator: [ operand, operand ]
			           }
		                ]
			      }

reduce

Optional object to get derived data by using a math operator. Can save calculation result to a temporary file or send to consume directly.

                           "reduce": {			        

expression

Required object expression that can contain a math operator.

	                        "expression": {       

fieldPath

Required string that specifies a field path in the event. Format is same as preceding field path definition, such as a.b.c.

If you want to do aggregation on name value pairs, you must use the following format.

"a.b[name=some-value].value"

For example, you want to do aggregation on the value of the name value pairs in the following name value pairs in orderLines.

{
    "values": [
        {
            "name": "amount",
            "value": 111
        }
    ],
    ....
}

The following example shows the fieldPath.

"orderLines.values[name=amount].value"

 	                             "fieldPath": ""

operator

Required string that specifies the math operator that you want to use.

The operator must be in list of "COUNT", "COUNT_DISTINCT", "SUM", "STD_DEVIATION_UPPER_BOUND", "STD_DEVIATION_LOWER_BOUND".

                                     "operator": ""
                                      }					  

The following example shows the operator "STD_DEVIATION_UPPER_BOUND", "STD_DEVIATION_LOWER_BOUND".

                            "operator": "",
                            "numberOfDeviationsInBound": 3

The numberOfDeviationsInBound parameter controls how many standard deviations from the mean to display. numberOfDeviationsInBound can be any nonnegative double, which means that you can request noninteger values, such as 1.5. A value of 0 is valid but returns the average for both upper and lower bounds. The upper and lower bounds are calculated as population metrics, so they are always the same as upper_population and lower_population respectively.

output_local

Required object that specifies the name of the result or fields that you want to save temporarily for the following steps:

Allowing optional string resultAlias and optional list fields in this object.

	                            "output_local": {
						"resultAlias":"",
						 "fields":[
							"field1",
							 ...
						     ]
					          } 				 

					}// end `reduce`
				 }// end `QUERY`
			}// end `expression`	

breakOnMatch

Optional Boolean value that indicates whether to break out of the map list after the current sequence is run. Default value is false.

			"breakOnMatch":
			},

nestedUpserts

Optional object to handle the case that needs to generate derived data after multiple line-level events are looped.

	        { 
			"nestedUpserts": {                        

nestedPath

Required string that specifies a nested path with an array. Format is same as the preceding field path definition.

For example, a.b.orderLines.

 	                 "nestedPath": “” 

object

Required object that specifies the fields that you want to get from the initial event.

You can specify the localIdentifiers field only in the sendObjectUpsertEvents.map.nestedUpserts.object level.

                         "objects": {
			      "localIdentifiers": {
			       "SELECT": "localIdentifier"
                                        }
				    }

map

Required array that specifies how to process event to get derived data. Definitions for all fields of sendObjectUpsertEvents.map.nestedUpserts.map are the same as sendObjectUpsertEvents.map.

			  "map": [
				   {

if

Optional object that specifies filter conditions. The definition of operator is same with an if event,

					 "if":{
					  } 

				      }
				  ] // end `sendObjectUpsertEvents.nestedUpserts.map`
			    } // end `nestedUpserts`
			} // end `sendObjectUpsertEvents.map[1]`
		   ] // end `sendObjectUpsertEvents.map`
	}// end sendObjectUpsertEvents
} end