FOUR-31892: Optimize Tasks API index endpoint query performance

ProcessMaker/Http/Controllers/Api/TaskController.php

@@ -16,6 +16,7 @@
 use ProcessMaker\Events\ActivityReassignment;
 use ProcessMaker\Facades\WorkflowManager;
 use ProcessMaker\Filters\Filter;
+use ProcessMaker\Http\Controllers\Api\V1_1\TaskController as V1_1TaskController;
 use ProcessMaker\Http\Controllers\Controller;
 use ProcessMaker\Http\Resources\ApiResource;
 use ProcessMaker\Http\Resources\Task as Resource;
@@ -126,6 +127,9 @@ class TaskController extends Controller
      */
     public function index(Request $request, $getTotal = false, User $user = null)
     {
+        if (config('app.processmaker_optimized_tasks_enabled')) {
+            return (new V1_1TaskController())->indexOptimized($request, $getTotal, $user);
+        }
         // If a specific user is specified, use it; otherwise use the authorized user
         // This is necessary to produce accurate counts for Saved Searches
         if (!$user) {

ProcessMaker/Http/Controllers/Api/V1_1/TaskController.php

@@ -4,20 +4,33 @@
 
 namespace ProcessMaker\Http\Controllers\Api\V1_1;
 
+use Carbon\Carbon;
 use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Database\QueryException;
 use Illuminate\Http\Request;
 use Illuminate\Support\Facades\Auth;
+use Illuminate\Support\Facades\Log;
 use ProcessMaker\Cache\Screens\ScreenCacheFactory;
 use ProcessMaker\Http\Controllers\Controller;
+use ProcessMaker\Http\Resources\TaskCollection;
 use ProcessMaker\Http\Resources\V1_1\TaskInterstitialResource;
 use ProcessMaker\Http\Resources\V1_1\TaskResource;
 use ProcessMaker\Http\Resources\V1_1\TaskScreen;
 use ProcessMaker\Models\ProcessRequest;
 use ProcessMaker\Models\ProcessRequestToken;
+use ProcessMaker\Models\User;
 use ProcessMaker\ProcessTranslations\TranslationManager;
+use ProcessMaker\Traits\TaskControllerIndexMethods;
 
 class TaskController extends Controller
 {
+    use TaskControllerIndexMethods;
+
+    public $doNotSanitize = [
+        'data',
+        'pmql',
+    ];
+
     protected $defaultFields = [
         'id',
         'element_id',
@@ -29,6 +42,56 @@ class TaskController extends Controller
         'process_request_id',
     ];
 
+    public function indexOptimized(Request $request, $getTotal = false, ?User $user = null)
+    {
+        if (!$user) {
+            $user = Auth::user();
+        }
+
+        $request->merge(['optimized' => true]);
+
+        $query = $this->indexOptimizedBaseQuery($request);
+        $this->applyIndexFieldSelection($query, $request);
+        $this->applyFilters($query, $request);
+        $this->excludeNonVisibleTasks($query, $request);
+        $this->applyColumnOrdering($query, $request);
+        $this->applyStatusFilter($query, $request);
+
+        if ($request->input('processesIManage') === 'true') {
+            $this->applyProcessManager($query, $user, $request);
+        } else {
+            $this->applyForCurrentUser($query, $user);
+        }
+
+        $this->applyPmql($query, $request, $user);
+        $this->applyAdvancedFilter($query, $request);
+        $query->overdue($request->input('overdue'));
+
+        if ($getTotal === true) {
+            return $query->count();
+        }
+
+        try {
+            $response = $query->paginate($request->input('per_page', 10));
+        } catch (QueryException $e) {
+            return $this->handleQueryException($e);
+        }
+
+        $response = $this->applyUserFilter($response, $request, $user);
+
+        if ($response->total() > 0 && $request->input('processesIManage') === 'true') {
+            $this->enableUserManager($user);
+        }
+
+        $inOverdueQuery = ProcessRequestToken::query()
+            ->whereIn('id', $response->pluck('id'))
+            ->where('due_at', '<', Carbon::now());
+
+        $response->inOverdue = $inOverdueQuery->count();
+
+        return new TaskCollection($response);
+    }
+
     /**
      * Display a listing of the resource.
      */
@@ -151,4 +214,23 @@ public function showInterstitial($taskId)
 
         return $response;
     }
+
+    private function handleQueryException(QueryException $e)
+    {
+        $regex = '~Column not found: 1054 Unknown column \'(.*?)\' in \'where clause\'~';
+
+        preg_match($regex, $e->getMessage(), $m);
+
+        $message = __('PMQL Is Invalid.');
+
+        if (count($m) > 1) {
+            $message .= ' ' . __('Column not found: ') . '"' . $m[1] . '"';
+        }
+
+        \Log::error($e->getMessage());
+
+        return response([
+            'message' => $message,
+        ], 422);
+    }
 }

ProcessMaker/Traits/HideSystemResources.php

@@ -3,6 +3,7 @@
 namespace ProcessMaker\Traits;
 
 use Facades\ProcessMaker\Helpers\CachedSchema;
+use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Schema;
 use Illuminate\Support\Str;
 use ProcessMaker\Models\Process;
@@ -57,7 +58,7 @@ public function scopeSystem($query)
         }
     }
 
-    public function scopeNonSystem($query)
+    public function scopeNonSystem($query, $optimized = false)
     {
         if (substr(static::class, -8) === 'Category') {
             return $query->where('is_system', false);
@@ -105,9 +106,25 @@ public function scopeNonSystem($query)
         } elseif (static::class == User::class) {
             return $query->where('is_system', false);
         } elseif (static::class === ProcessRequestToken::class) {
-            return $query->whereHas('process.categories', function ($query) {
-                $query->where('is_system', false);
-            });
+            // Direct EXISTS avoids Process global scopes (e.g. published/process_versions).
+            if (!$optimized) {
+                return $query->whereHas('process.categories', function ($query) {
+                    $query->where('is_system', false);
+                });
+            } else {
+                return $query->whereExists(function ($sub) {
+                    $sub->select(DB::raw(1))
+                        ->from('processes')
+                        ->join('category_assignments', function ($join) {
+                            $join->on('category_assignments.assignable_id', '=', 'processes.id')
+                                ->where('category_assignments.assignable_type', '=', Process::class);
+                        })
+                        ->join('process_categories', 'process_categories.id', '=', 'category_assignments.category_id')
+                        ->whereColumn('processes.id', 'process_request_tokens.process_id')
+                        ->whereNull('processes.deleted_at')
+                        ->where('process_categories.is_system', false);
+                });
+            }
         } elseif (static::class === ProcessTemplates::class) {
             return $query->where('process_templates.is_system', false)
                 ->when(CachedSchema::hasColumn('process_templates', 'asset_type'), function ($query) {

ProcessMaker/Traits/TaskControllerIndexMethods.php

@@ -21,6 +21,21 @@ trait TaskControllerIndexMethods
 {
     private const SELF_SERVICE_STATUS = 'self service';
 
+    private const INDEX_JSON_COLUMNS = ['data', 'self_service_groups', 'token_properties'];
+
+    private const INDEX_COLUMN_FIELD_MAP = [
+        'case_number' => ['process_request_id'],
+        'case_title' => ['process_request_id'],
+        'is_priority' => ['is_priority'],
+        'element_name' => ['element_name', 'element_type'],
+        'status' => ['status', 'is_self_service', 'due_at'],
+        'due_at' => ['due_at'],
+        'completed_at' => ['completed_at'],
+        'process' => ['process_id'],
+        'assignee' => ['user_id', 'is_self_service'],
+        'request' => ['process_request_id', 'process_id'],
+    ];
+
     private function indexBaseQuery($request)
     {
         // Parse the includes parameter
@@ -62,6 +77,122 @@ private function indexBaseQuery($request)
         return $query;
     }
 
+    private function indexOptimizedBaseQuery($request)
+    {
+        $includes = $request->has('include')
+            ? array_map('trim', explode(',', $request->input('include')))
+            : [];
+        $includeData = in_array('data', $includes, true);
+
+        $query = ProcessRequestToken::query();
+        $with = [];
+
+        if (in_array('processRequest', $includes, true)) {
+            $processRequestColumns = [
+                'id',
+                'uuid',
+                'case_number',
+                'case_title',
+                'name',
+                'status',
+                'user_id',
+                'process_id',
+                'parent_id',
+            ];
+            $with['processRequest'] = function ($q) use ($includeData, $processRequestColumns) {
+                $q->select($processRequestColumns);
+                if (!$includeData) {
+                    $q->exclude(['data']);
+                }
+            };
+        }
+
+        if (in_array('process', $includes, true)) {
+            $with['process'] = fn ($q) => $q->select(['id', 'name', 'uuid']);
+        }
+
+        if (in_array('user', $includes, true)) {
+            $with['user'] = fn ($q) => $q->select(['id', 'firstname', 'lastname', 'avatar', 'status']);
+        }
+
+        if (in_array('draft', $includes, true)) {
+            $with['draft'] = fn ($q) => $q->select(['id', 'uuid', 'process_request_token_id']);
+        }
+
+        $handledIncludes = ['data', 'processRequest', 'process', 'user', 'draft', 'processRequest.process'];
+        $additionalIncludes = array_values(array_diff($includes, $handledIncludes));
+
+        if (!empty($with)) {
+            $query->with($with);
+        }
+
+        if (!empty($additionalIncludes)) {
+            $query->with($additionalIncludes);
+        }
+
+        return $query;
+    }
+
+    private function resolveIndexFields($request): ?array
+    {
+        $fields = $request->input('fields', '');
+        if ($fields) {
+            $selectedFields = array_filter(array_map('trim', explode(',', $fields)));
+        } else {
+            $columns = $request->input('columns', '');
+            if (!$columns) {
+                return null;
+            }
+            $selectedFields = $this->mapColumnsToFields(array_map('trim', explode(',', $columns)));
+        }
+
+        if (!in_array('id', $selectedFields, true)) {
+            $selectedFields[] = 'id';
+        }
+
+        return array_values(array_unique($selectedFields));
+    }
+
+    private function mapColumnsToFields(array $columns): array
+    {
+        $fields = [];
+
+        foreach ($columns as $column) {
+            if (str_starts_with($column, 'data.')) {
+                continue;
+            }
+
+            if (isset(self::INDEX_COLUMN_FIELD_MAP[$column])) {
+                $fields = array_merge($fields, self::INDEX_COLUMN_FIELD_MAP[$column]);
+                continue;
+            }
+
+            if (in_array($column, ['draft', 'actions'], true)) {
+                continue;
+            }
+
+            $fields[] = $column;
+        }
+
+        return array_values(array_unique(array_merge(
+            $fields,
+            ['id', 'process_id', 'process_request_id']
+        )));
+    }
+
+    private function applyIndexFieldSelection($query, $request): void
+    {
+        $selectedFields = $this->resolveIndexFields($request);
+
+        if ($selectedFields !== null) {
+            $query->select($selectedFields);
+
+            return;
+        }
+
+        $query->exclude(self::INDEX_JSON_COLUMNS);
+    }
+
     private function applyFilters($query, $request)
     {
         $filter = $request->input('filter', '');
@@ -160,6 +291,7 @@ private function addTaskData($response)
     private function excludeNonVisibleTasks($query, $request)
     {
         $nonSystem = filter_var($request->input('non_system'), FILTER_VALIDATE_BOOLEAN);
+        $optimized = filter_var($request->input('optimized'), FILTER_VALIDATE_BOOLEAN);
         $allTasks = filter_var($request->input('all_tasks'), FILTER_VALIDATE_BOOLEAN);
         $hitlEnabled = filter_var(config('smart-extract.hitl_enabled'), FILTER_VALIDATE_BOOLEAN);
         $includeScreen = filter_var($request->input('includeScreen'), FILTER_VALIDATE_BOOLEAN);
@@ -177,15 +309,15 @@ private function excludeNonVisibleTasks($query, $request)
                 });
             });
         })
-            ->when($nonSystem, function ($query) use ($hitlEnabled) {
+            ->when($nonSystem, function ($query) use ($hitlEnabled, $optimized) {
                 if (!$hitlEnabled) {
-                    $query->nonSystem();
+                    $query->nonSystem($optimized);
 
                     return;
                 }
 
-                $query->where(function ($query) {
-                    $query->nonSystem();
+                $query->where(function ($query) use ($optimized) {
+                    $query->nonSystem($optimized);
                     $query->orWhere(function ($query) {
                         $query->where('element_type', '=', 'task');
                         $query->where('element_name', '=', 'Manual Document Review');

config/app.php

@@ -79,6 +79,9 @@
     // System-level scripts timeout
     'processmaker_system_scripts_timeout_seconds' => env('PROCESSMAKER_SYSTEM_SCRIPTS_TIMEOUT_SECONDS', 300),
 
+    // Enable optimized tasks
+    'processmaker_optimized_tasks_enabled' => env('OPTIMIZED_TASKS_ENABLED', true),
+
     // Since the task scheduler has a preset of one minute (crontab), the times
     // must be rounded or truncated to the nearest HH:MM:00 before compare
     'timer_events_seconds' => env('TIMER_EVENTS_SECONDS', 'truncate'),

routes/v1_1/api.php

@@ -12,6 +12,10 @@
     ->group(function () {
         // Tasks Endpoints
         Route::name('tasks.')->prefix('tasks')->group(function () {
+            // Route to list optimized tasks
+            Route::get('/tasksOptimized', [TaskController::class, 'indexOptimized'])
+                ->name('indexOptimized');
+
             // Route to list tasks
             Route::get('/', [TaskController::class, 'index'])
                 ->name('index');