php - Laravel Dealing With API Rate Limits in Queued Jobs

one text

My application is communicating with 3rd party API to get a list of properties for real estate agencies from CRM.

When I am trying to download a number of images for each property, the API blocks the request to download images after it hits the rate limit. After pausing for around 15 to 20 seconds, the server again allows me to download more images.

In order to achieve the download of property images for each property, I have implemented a Laravel job to process it on a Queue.

If we hit a rate limit, the media URL does not open and the UnreachableUrl exception is thrown by Spatie Media Library, I am debugging that unreachable message and manually releasing the job back to the queue to retry after 20 seconds (which is the API server reset time).
However, after the 2nd and 3rd, I got Max Attempts Exceeded Exception, as well as job failed() exception message.

I would like to only Log the final failed job exception as an error, not the Max Attempts Exceeded Exception. Or is there any convenient way of achieving this process and logging the error only if the final attempt failed not the intermediate process?

Thanks in advance.

** Class to create each property downloaded from API and loop through each media file and dispatch job to download image **

class PropertImportService {

   /**
    * @var PropertyRepository
    */
   protected $propertyRepository;

   public function importProperties() {
       $apiService = new APIService();
       $response = $apiService->importAgencyProperties();
       $importedProperties = $response['imported_properties'];
       foreach ($importedProperties as $imported_property) {

           // DB ENTRY for PROPERTY
           $property_data = $this->preparePropertyDataArray($imported_property);
           $property = $this->propertyRepository->create($property_data);

           // DOWNLOAD MEDIA FILES
           $media = $imported_property->getMedia();
           if(isset($media) && isset($property)) {
               $this->saveMediaFiles($property, $media, $imported_property->getJsonId());
           }
       }

   }

   private function saveMediaFiles($property, $media, $json_id) {
       $maxImages = 5;
       $downloaded = 0;
       foreach ($media as $media_key => $media_value) {
           $media_url = $media_value['url'];
          
          // No. of processes for default queue is 1.
           DownloadMedia::dispatch($property, $media_url, $media_collection = 'default')->onQueue('default');

           $downloaded++;
           if($downloaded >= $maxImages){
               break;
           }
       }
   }
}

** DownloadMedia Job to process media download **

<?php

namespace App\Jobs\MediaLibrary;

use App\Jobs\Processors\ProcessDownloadMedia;
use App\Models\Property\Property;
use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\MaxAttemptsExceededException;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Spatie\MediaLibrary\Exceptions\FileCannotBeAdded\UnreachableUrl;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Contracts\Queue\ShouldQueue;
use Throwable;

class DownloadMedia implements ShouldQueue
{
   use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

   /**
    * The number of times the job may be attempted.
    *
    * @var int
    */
   public $tries = 2;

   /**
    * The number of seconds to wait before retrying the job.
    *
    * @var int
    */
   public $backoff = 10;

   /**
    * The maximum number of unhandled exceptions to allow before failing.
    *
    * @var int
    */
   public $maxExceptions = 3;

   /**
    * Create a model.
    */
   protected $model;
   /**
    * Media URL of imported property.
    */
   protected $media_url;

   /**
    * Media collection.
    */
   protected $media_collection;

   public function __construct(Property $property, $media_url, $media_collection)
   {
       $this->model = $property;
       $this->media_url = $media_url;
       $this->media_collection = $media_collection;
   }

   public function handle(ProcessDownloadMedia $processor)
   {
       try{
           $processor->downloadMedia($this->model, $this->media_url, $this->media_collection);
       } catch(UnreachableUrl $e) {
           Log::debug('UnreachableUrl Exception while downloading media from URL:'. $this->media_url);
           //throw new \Exception('failed');
           $this->release(20);
       } catch(Exception $e) {
           Log::error('\nException while downloading media from URL:'. $this->media_url . 'Error Message: '.$e->getMessage().'\n');
       }
   }

   /**
    * The job failed to process.
    *
    * @param  Exception  $exception
    * @return void
    */
   public function failed(Exception $exception)
   {
       Log::error('\nException while downloading media from URL:'. $this->media_url . 'Error Message: '. $exception->getMessage().'\n');
   }
}

** Job processor to save media using Spatie media library **

<?php

namespace App\Jobs\Processors;

use Exception;
use Spatie\MediaLibrary\Exceptions\FileCannotBeAdded\UnreachableUrl;
use Illuminate\Support\Facades\Log;

class ProcessDownloadMedia
{
   function downloadMedia($model, $media_url, $media_collection) {
       $media_url_without_query_string = preg_split("/[?#]/", $media_url)[0];
       $fileinfo = pathinfo($media_url_without_query_string);
       $file_extension = (array_key_exists('extension',$fileinfo) && $fileinfo['extension']!='')?$fileinfo['extension']:'jpg';
       $filename = $fileinfo['filename'] .'.'.$file_extension;

       $model->addMediaFromUrl($media_url)
           ->usingFileName($filename)
           ->withCustomProperties(['orgUrl' => $media_url])
           ->toMediaCollection($media_collection);
   }

}