patterns-data-processing

name	patterns-data-processing
description	Mendix data processing patterns: loops, aggregates (count/sum/average/min/max), list operations, batch processing, and association retrieval
compatibility	opencode

Data processing pattern expert for Mendix. Covers loops, aggregates, list operations, batch processing, association retrieval, and data transformation microflows.

Patterns for loops, aggregates, batch processing, and data transformation.

- Iterate over a list of entities in a microflow - Calculate count, sum, average, min, or max - Process data in batches - Retrieve related objects via association - Transform or copy entity data

<loop_patterns>

Basic Loop

/**
 * Process all items in a list
 */
create microflow Module.ProcessItems (
  $Items: list of Module.Item
)
returns boolean
begin
  declare $ProcessedCount integer = 0;

  loop $item in $Items
  begin
    change $item (ProcessedDate = [%CurrentDateTime%]);
    commit $item;
    set $ProcessedCount = $ProcessedCount + 1;
  end loop;

  log info node 'Processing' 'Processed ' + $ProcessedCount + ' items';
  return true;
end;
/

Loop with Filtering

create microflow Module.ProcessActiveItems (
  $Items: list of Module.Item
)
returns integer
begin
  declare $count integer = 0;

  loop $item in $Items
  begin
    if $item/IsActive then
      change $item (LastProcessed = [%CurrentDateTime%]);
      commit $item;
      set $count = $count + 1;
    end if;
  end loop;

  return $count;
end;
/

Loop with Accumulator

create microflow Module.CalculateOrderTotal (
  $Orders: list of Module.Order
)
returns decimal
begin
  declare $Total decimal = 0;

  loop $Order in $Orders
  begin
    set $Total = $Total + $Order/Amount;
  end loop;

  return $Total;
end;
/

</loop_patterns>

<retrieve_by_association> Use retrieve $list from $Parent/Module.AssociationName to retrieve related objects via association instead of a database XPath query. This is required for:

• Non-persistent entities (NPEs) — database XPath queries always return empty for NPEs
• Uncommitted objects — objects not yet committed to the database
• JSON mapping results — imported data structures held in memory

Persistent Entity Example

create microflow Module.GetCustomerOrders (
  $Customer : Module.Customer
)
returns list of Module.Order
begin
  retrieve $Orders from $Customer/Module.Order_Customer;
  return $Orders;
end;
/

Non-Persistent Entity Example (NPE)

/**
 * Process imported rows from an in-memory result object.
 * Database RETRIEVE would return empty for NPEs — use association retrieve.
 */
create microflow Module.ProcessImportRows (
  $ImportResult : Module.ImportResult
)
returns integer
begin
  retrieve $Rows from $ImportResult/Module.ImportResult_ImportRow;

  declare $ValidCount integer = 0;

  loop $row in $Rows
  begin
    if $row/IsValid then
      set $ValidCount = $ValidCount + 1;
    end if;
  end loop;

  return $ValidCount;
end;
/

When to Use Which Retrieve

Scenario	Syntax	Why
Query persistent entities by attribute	retrieve $list from Module.Entity where ...	Database XPath query
Get related persistent objects	retrieve $list from $Parent/Module.Association	Simpler, no XPath needed
Get related NPEs / uncommitted objects	retrieve $list from $Parent/Module.Association	Only option — database has no data
JSON mapping results (import)	retrieve $list from $Parent/Module.Association	Mapping creates in-memory NPEs

Important: Association retrieve always returns a list. It does not support WHERE, SORT BY, LIMIT, or OFFSET clauses. </retrieve_by_association>

<aggregate_patterns> Aggregates use function-call syntax — there is no AGGREGATE keyword.

Function	Syntax	Returns
COUNT	$n = count($list)	Integer
SUM	$n = sum($list.Attr)	Decimal
AVERAGE	$n = average($list.Attr)	Decimal
MINIMUM	$n = minimum($list.Attr)	Same as attribute
MAXIMUM	$n = maximum($list.Attr)	Same as attribute

Important: RETRIEVE implicitly declares its variable — do NOT add a separate DECLARE before RETRIEVE, or you'll get CE0111 "Duplicate variable name".

Count Items

create microflow Module.CountActiveCustomers ()
returns integer
begin
  retrieve $Customers from Module.Customer
    where IsActive = true;

  $count = count($Customers);
  return $count;
end;
/

Sum Values

create microflow Module.GetCustomerTotalOrders (
  $Customer: Module.Customer
)
returns decimal
begin
  retrieve $Orders from Module.Order
    where Module.Order_Customer = $Customer;

  $Total = sum($Orders.Amount);
  return $Total;
end;
/

Min/Max

$MinPrice = minimum($Products.Price);
$MaxPrice = maximum($Products.Price);

</aggregate_patterns>

<list_operations>

Add to List

create microflow Module.CollectHighValueOrders (
  $Orders: list of Module.Order,
  $Threshold: decimal
)
returns list of Module.Order
begin
  declare $HighValue list of Module.Order = empty;

  loop $Order in $Orders
  begin
    if $Order/Amount > $Threshold then
      add $Order to $HighValue;
    end if;
  end loop;

  return $HighValue;
end;
/

Remove from List

create microflow Module.FilterActiveItems (
  $Items: list of Module.Item
)
returns list of Module.Item
begin
  declare $ToRemove list of Module.Item = empty;

  loop $item in $Items
  begin
    if not $item/IsActive then
      add $item to $ToRemove;
    end if;
  end loop;

  loop $item in $ToRemove
  begin
    remove $item from $Items;
  end loop;

  return $Items;
end;
/

</list_operations>

<batch_processing>

/**
 * Process large dataset in batches
 * Commits after each batch to avoid memory issues
 */
create microflow Module.BatchProcess (
  $Items: list of Module.Item,
  $BatchSize: integer
)
returns integer
begin
  declare $Processed integer = 0;
  declare $BatchCount integer = 0;

  loop $item in $Items
  begin
    change $item (status = 'Processed');

    set $BatchCount = $BatchCount + 1;
    set $Processed = $Processed + 1;

    if $BatchCount >= $BatchSize then
      commit $item;
      set $BatchCount = 0;
      log info node 'Batch' 'Processed ' + $Processed + ' items';
    end if;
  end loop;

  return $Processed;
end;
/

</batch_processing>

<data_transformation>

Copy Entity

create microflow Module.CopyOrder (
  $source: Module.Order
)
returns Module.Order
begin
  declare $Copy as Module.Order;

  $Copy = create Module.Order (
    OrderNumber = 'COPY-' + $source/OrderNumber,
    Amount = $source/Amount,
    status = 'Draft',
    CreatedDate = [%CurrentDateTime%]
  );

  set $Copy/Module.Order_Customer = $source/Module.Order_Customer;
  commit $Copy;
  return $Copy;
end;
/

Transform List

create microflow Module.CreateOrderSummaries (
  $Orders: list of Module.Order
)
returns list of Module.OrderSummary
begin
  declare $Summaries list of Module.OrderSummary = empty;
  declare $Summary as Module.OrderSummary;

  loop $Order in $Orders
  begin
    $Summary = create Module.OrderSummary (
      OrderNumber = $Order/OrderNumber,
      TotalAmount = $Order/Amount,
      CustomerName = $Order/Module.Order_Customer/Name
    );
    add $Summary to $Summaries;
  end loop;

  return $Summaries;
end;
/

</data_transformation>

<best_practices>

1. Commit inside loops carefully: Can cause performance issues on large sets
2. Use batch commits: Commit every N records for large datasets
3. Log progress: Add logging for long-running operations
4. Handle errors gracefully: Don't let one bad record stop the whole process
5. Return counts: Help callers know what was processed
6. Use meaningful variable names: $ProcessedCount not $c </best_practices>

<output_rules>Output MDL code only in code blocks. Keep explanations concise.</output_rules>